From 9535a581641cedddff20ba22dfa092383d3a7e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 16 May 2022 16:41:54 +0200 Subject: [PATCH] Remove `libbeat/keystore`, `libbeat/common/kubernetes` and `libbeat/common/docker` (#31607) --- NOTICE.txt | 243 ++++- filebeat/autodiscover/builder/hints/logs.go | 2 +- .../autodiscover/builder/hints/logs_test.go | 2 +- go.mod | 17 +- go.sum | 31 +- .../autodiscover/builder/hints/monitors.go | 2 +- .../builder/hints/monitors_test.go | 2 +- libbeat/autodiscover/appender.go | 2 +- libbeat/autodiscover/appender_test.go | 2 +- .../autodiscover/appenders/config/config.go | 2 +- .../appenders/config/config.go.orig | 113 --- .../appenders/config/config_test.go | 2 +- libbeat/autodiscover/autodiscover.go | 4 +- libbeat/autodiscover/autodiscover_test.go | 4 +- libbeat/autodiscover/builder.go | 9 +- libbeat/autodiscover/builder_test.go | 3 +- libbeat/autodiscover/eventselect.go | 2 +- libbeat/autodiscover/provider.go | 4 +- .../autodiscover/providers/docker/config.go | 2 +- .../autodiscover/providers/docker/docker.go | 6 +- .../docker/docker_integration_test.go | 4 +- .../providers/docker/docker_test.go | 4 +- .../providers/jolokia/discovery.go | 2 +- .../autodiscover/providers/jolokia/jolokia.go | 4 +- .../providers/kubernetes/config.go | 7 +- .../providers/kubernetes/config_test.go | 2 +- .../providers/kubernetes/kubernetes.go | 8 +- .../autodiscover/providers/kubernetes/node.go | 6 +- .../providers/kubernetes/node_test.go | 6 +- .../autodiscover/providers/kubernetes/pod.go | 6 +- .../providers/kubernetes/pod_test.go | 6 +- .../providers/kubernetes/service.go | 6 +- .../providers/kubernetes/service_test.go | 6 +- libbeat/autodiscover/template/config.go | 8 +- libbeat/autodiscover/template/config_test.go | 6 +- libbeat/beat/beat.go | 2 +- libbeat/cmd/instance/beat.go | 4 +- libbeat/cmd/keystore.go | 2 +- libbeat/common/bus/bus.go | 156 ---- libbeat/common/bus/bus_test.go | 121 --- libbeat/common/docker/client.go | 54 -- libbeat/common/docker/client_test.go | 63 -- libbeat/common/docker/helpers.go | 60 -- libbeat/common/docker/watcher.go | 480 ---------- libbeat/common/docker/watcher_test.go | 523 ----------- libbeat/common/kubernetes/eventhandler.go | 231 ----- libbeat/common/kubernetes/informer.go | 310 ------- .../k8skeystore/kubernetes_keystore.go | 137 --- .../k8skeystore/kubernetes_keystore_test.go | 209 ----- libbeat/common/kubernetes/metadata/config.go | 62 -- .../common/kubernetes/metadata/metadata.go | 180 ---- .../common/kubernetes/metadata/namespace.go | 130 --- .../kubernetes/metadata/namespace_test.go | 155 ---- libbeat/common/kubernetes/metadata/node.go | 107 --- .../common/kubernetes/metadata/node_test.go | 157 ---- libbeat/common/kubernetes/metadata/pod.go | 196 ---- .../common/kubernetes/metadata/pod_test.go | 845 ------------------ .../common/kubernetes/metadata/resource.go | 183 ---- .../kubernetes/metadata/resource_test.go | 128 --- libbeat/common/kubernetes/metadata/service.go | 113 --- .../kubernetes/metadata/service_test.go | 330 ------- libbeat/common/kubernetes/options.go | 23 - libbeat/common/kubernetes/types.go | 154 ---- libbeat/common/kubernetes/util.go | 335 ------- libbeat/common/kubernetes/util_test.go | 323 ------- libbeat/common/kubernetes/watcher.go | 266 ------ libbeat/keystore/config.go | 27 - libbeat/keystore/file_keystore.go | 457 ---------- libbeat/keystore/file_keystore_test.go | 343 ------- libbeat/keystore/keystore.go | 123 --- libbeat/keystore/keystore_test.go | 52 -- libbeat/keystore/secure_string.go | 49 - libbeat/keystore/secure_string_test.go | 55 -- .../add_docker_metadata.go | 2 +- .../add_docker_metadata_integration_test.go | 2 +- .../add_docker_metadata_test.go | 4 +- .../processors/add_docker_metadata/config.go | 2 +- .../add_kubernetes_metadata/config.go | 4 +- .../add_kubernetes_metadata/indexers.go | 7 +- .../add_kubernetes_metadata/indexers_test.go | 5 +- .../add_kubernetes_metadata/kubernetes.go | 4 +- libbeat/tests/compose/wrapper.go | 2 +- libbeat/tests/docker/docker.go | 4 +- .../appender/kubernetes/token/token.go | 2 +- .../appender/kubernetes/token/token_test.go | 2 +- .../autodiscover/builder/hints/metrics.go | 2 +- .../builder/hints/metrics_test.go | 4 +- metricbeat/module/docker/container/data.go | 2 +- metricbeat/module/docker/docker.go | 2 +- .../docker/event/event_integration_test.go | 2 +- metricbeat/module/docker/helper.go | 2 +- metricbeat/module/docker/helper_test.go | 2 +- metricbeat/module/docker/image/data.go | 2 +- metricbeat/module/kubernetes/event/config.go | 2 +- metricbeat/module/kubernetes/event/event.go | 2 +- metricbeat/module/kubernetes/node/node.go | 2 +- metricbeat/module/kubernetes/pod/pod.go | 2 +- .../kubernetes/state_cronjob/state_cronjob.go | 2 +- .../state_daemonset/state_daemonset.go | 2 +- .../state_deployment/state_deployment.go | 2 +- .../module/kubernetes/state_job/state_job.go | 2 +- .../kubernetes/state_node/state_node.go | 2 +- .../state_persistentvolume.go | 2 +- .../state_persistentvolumeclaim.go | 2 +- .../module/kubernetes/state_pod/state_pod.go | 2 +- .../state_replicaset/state_replicaset.go | 2 +- .../kubernetes/state_service/state_service.go | 2 +- .../state_statefulset/state_statefulset.go | 2 +- .../module/kubernetes/util/kubernetes.go | 11 +- .../module/kubernetes/util/kubernetes_test.go | 2 +- metricbeat/module/kubernetes/volume/volume.go | 5 +- .../pipelinemanager/clientLogReader.go | 2 +- x-pack/functionbeat/manager/core/makezip.go | 2 +- .../providers/aws/ec2/provider.go | 4 +- .../providers/aws/ec2/provider_test.go | 4 +- .../providers/aws/elb/provider.go | 4 +- .../providers/aws/elb/provider_test.go | 4 +- .../providers/aws/test/provider.go | 2 +- .../autodiscover/providers/nomad/nomad.go | 4 +- .../providers/nomad/nomad_test.go | 2 +- .../module/awsfargate/task_stats/container.go | 6 +- 121 files changed, 392 insertions(+), 7428 deletions(-) delete mode 100644 libbeat/autodiscover/appenders/config/config.go.orig delete mode 100644 libbeat/common/bus/bus.go delete mode 100644 libbeat/common/bus/bus_test.go delete mode 100644 libbeat/common/docker/client.go delete mode 100644 libbeat/common/docker/client_test.go delete mode 100644 libbeat/common/docker/helpers.go delete mode 100644 libbeat/common/docker/watcher.go delete mode 100644 libbeat/common/docker/watcher_test.go delete mode 100644 libbeat/common/kubernetes/eventhandler.go delete mode 100644 libbeat/common/kubernetes/informer.go delete mode 100644 libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go delete mode 100644 libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go delete mode 100644 libbeat/common/kubernetes/metadata/config.go delete mode 100644 libbeat/common/kubernetes/metadata/metadata.go delete mode 100644 libbeat/common/kubernetes/metadata/namespace.go delete mode 100644 libbeat/common/kubernetes/metadata/namespace_test.go delete mode 100644 libbeat/common/kubernetes/metadata/node.go delete mode 100644 libbeat/common/kubernetes/metadata/node_test.go delete mode 100644 libbeat/common/kubernetes/metadata/pod.go delete mode 100644 libbeat/common/kubernetes/metadata/pod_test.go delete mode 100644 libbeat/common/kubernetes/metadata/resource.go delete mode 100644 libbeat/common/kubernetes/metadata/resource_test.go delete mode 100644 libbeat/common/kubernetes/metadata/service.go delete mode 100644 libbeat/common/kubernetes/metadata/service_test.go delete mode 100644 libbeat/common/kubernetes/options.go delete mode 100644 libbeat/common/kubernetes/types.go delete mode 100644 libbeat/common/kubernetes/util.go delete mode 100644 libbeat/common/kubernetes/util_test.go delete mode 100644 libbeat/common/kubernetes/watcher.go delete mode 100644 libbeat/keystore/config.go delete mode 100644 libbeat/keystore/file_keystore.go delete mode 100644 libbeat/keystore/file_keystore_test.go delete mode 100644 libbeat/keystore/keystore.go delete mode 100644 libbeat/keystore/keystore_test.go delete mode 100644 libbeat/keystore/secure_string.go delete mode 100644 libbeat/keystore/secure_string_test.go diff --git a/NOTICE.txt b/NOTICE.txt index 2cf3eab1350..a4bcb4e99c4 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4638,11 +4638,11 @@ third-party archives. -------------------------------------------------------------------------------- Dependency : github.com/docker/docker -Version: v20.10.7+incompatible +Version: v20.10.12+incompatible Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/docker/docker@v20.10.7+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/docker/docker@v20.10.12+incompatible/LICENSE: Apache License @@ -5835,6 +5835,217 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/elastic-agent-autodiscover +Version: v0.1.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.1.1/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 Version: v7.0.0-20210727140539-f0905d9377f6 @@ -16457,11 +16668,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.0.0-20211216030914-fe4d6282115f +Version: v0.0.0-20220225172249-27dd8689420f Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20211216030914-fe4d6282115f/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.0.0-20220225172249-27dd8689420f/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -18012,11 +18223,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : k8s.io/api -Version: v0.23.1 +Version: v0.23.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/k8s.io/api@v0.23.1/LICENSE: +Contents of probable licence file $GOMODCACHE/k8s.io/api@v0.23.4/LICENSE: Apache License @@ -18224,11 +18435,11 @@ Contents of probable licence file $GOMODCACHE/k8s.io/api@v0.23.1/LICENSE: -------------------------------------------------------------------------------- Dependency : k8s.io/apimachinery -Version: v0.23.1 +Version: v0.23.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/k8s.io/apimachinery@v0.23.1/LICENSE: +Contents of probable licence file $GOMODCACHE/k8s.io/apimachinery@v0.23.4/LICENSE: Apache License @@ -18436,11 +18647,11 @@ Contents of probable licence file $GOMODCACHE/k8s.io/apimachinery@v0.23.1/LICENS -------------------------------------------------------------------------------- Dependency : k8s.io/client-go -Version: v0.23.1 +Version: v0.23.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/k8s.io/client-go@v0.23.1/LICENSE: +Contents of probable licence file $GOMODCACHE/k8s.io/client-go@v0.23.4/LICENSE: Apache License @@ -36095,11 +36306,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.0.0-20210615171337-6886f2dfbf5b +Version: v0.0.0-20210927222741-03fcf44c2211 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.0.0-20210615171337-6886f2dfbf5b/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.0.0-20210927222741-03fcf44c2211/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -37996,11 +38207,11 @@ Contents of probable licence file $GOMODCACHE/k8s.io/kube-openapi@v0.0.0-2021111 -------------------------------------------------------------------------------- Dependency : k8s.io/utils -Version: v0.0.0-20210930125809-cb0fa318a74b +Version: v0.0.0-20211116205334-6203023598ed Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/k8s.io/utils@v0.0.0-20210930125809-cb0fa318a74b/LICENSE: +Contents of probable licence file $GOMODCACHE/k8s.io/utils@v0.0.0-20211116205334-6203023598ed/LICENSE: Apache License @@ -38862,11 +39073,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : sigs.k8s.io/structured-merge-diff/v4 -Version: v4.1.2 +Version: v4.2.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/sigs.k8s.io/structured-merge-diff/v4@v4.1.2/LICENSE: +Contents of probable licence file $GOMODCACHE/sigs.k8s.io/structured-merge-diff/v4@v4.2.1/LICENSE: Apache License Version 2.0, January 2004 diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index ddec4c027d0..5ab37ab6f24 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -21,6 +21,7 @@ import ( "fmt" "regexp" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-ucfg" @@ -31,7 +32,6 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 9546adb8ce3..a8bba01d52b 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/paths" diff --git a/go.mod b/go.mod index db8f45e2dcf..561d70c1421 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/denisenkom/go-mssqldb v0.9.0 github.com/dgraph-io/badger/v3 v3.2103.1 github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 - github.com/docker/docker v20.10.7+incompatible + github.com/docker/docker v20.10.12+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/go-plugins-helpers v0.0.0-20181025120712-1e6269c305b8 github.com/docker/go-units v0.4.0 @@ -135,7 +135,7 @@ require ( golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.5.1 - golang.org/x/net v0.0.0-20211216030914-fe4d6282115f + golang.org/x/net v0.0.0-20220225172249-27dd8689420f golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20220209214540-3681064d5158 @@ -153,14 +153,15 @@ require ( gotest.tools v2.2.0+incompatible gotest.tools/gotestsum v1.7.0 howett.net/plist v1.0.0 - k8s.io/api v0.23.1 - k8s.io/apimachinery v0.23.1 - k8s.io/client-go v0.23.1 + k8s.io/api v0.23.4 + k8s.io/apimachinery v0.23.4 + k8s.io/client-go v0.23.4 kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 ) require ( github.com/elastic/bayeux v1.0.5 + github.com/elastic/elastic-agent-autodiscover v0.1.1 github.com/elastic/elastic-agent-libs v0.2.3 github.com/shirou/gopsutil/v3 v3.21.12 go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0 @@ -277,7 +278,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect @@ -287,10 +288,10 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/klog/v2 v2.30.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect + k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 3d473b487bf..8de205c2113 100644 --- a/go.sum +++ b/go.sum @@ -531,8 +531,9 @@ github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20170802015333-8af4db6f002a/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.12+incompatible h1:CEeNmFM0QZIsJCZKMkZx0ZcahTiewkrgiwfYD+dfl1U= +github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= @@ -569,8 +570,11 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= +github.com/elastic/elastic-agent-autodiscover v0.1.1 h1:oSV42MKIcPq30VJFZwc066eVm+pKpxfQNY4RIPE1fyg= +github.com/elastic/elastic-agent-autodiscover v0.1.1/go.mod h1:Gg1fsQI+rVms9FJ2DefBSojfPIzgkV8xlyG8fPG0DE8= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= +github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM= github.com/elastic/elastic-agent-libs v0.2.3 h1:GY8M0fxOs/GBY2nIB+JOB91aoD72S87iEcm2qVGFUqI= github.com/elastic/elastic-agent-libs v0.2.3/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= @@ -1394,6 +1398,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -2005,8 +2010,9 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2184,8 +2190,9 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2570,8 +2577,8 @@ k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= -k8s.io/api v0.23.1 h1:ncu/qfBfUoClqwkTGbeRqqOqBCRoUAflMuOaOD7J0c8= -k8s.io/api v0.23.1/go.mod h1:WfXnOnwSqNtG62Y1CdjoMxh7r7u9QXGCkA1u0na2jgo= +k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E= +k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= k8s.io/apimachinery v0.17.5/go.mod h1:ioIo1G/a+uONV7Tv+ZmCbMG1/a3kVw5YcDdncd8ugQ0= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= @@ -2579,8 +2586,8 @@ k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MA k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= -k8s.io/apimachinery v0.23.1 h1:sfBjlDFwj2onG0Ijx5C+SrAoeUscPrmghm7wHP+uXlo= -k8s.io/apimachinery v0.23.1/go.mod h1:SADt2Kl8/sttJ62RRsi9MIV4o8f5S3coArm0Iu3fBno= +k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM= +k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= @@ -2591,8 +2598,8 @@ k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= -k8s.io/client-go v0.23.1 h1:Ma4Fhf/p07Nmj9yAB1H7UwbFHEBrSPg8lviR24U2GiQ= -k8s.io/client-go v0.23.1/go.mod h1:6QSI8fEuqD4zgFK0xbdwfB/PthBsIxCJMa3s17WlcO0= +k8s.io/client-go v0.23.4 h1:YVWvPeerA2gpUudLelvsolzH7c2sFoXXR5wM/sWqNFU= +k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0= k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= @@ -2633,8 +2640,9 @@ k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 h1:2nmqI+aw7EQZuelYktkQHBE4jESD2tOR+lOJEnv/Apo= kernel.org/pub/linux/libs/security/libcap/cap v1.2.57/go.mod h1:uI99C3r4SXvJeuqoEtx/eWt7UbmfqqZ80H8q+9t/A7I= kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 h1:NOFATXSf5z/cMR3HIwQ3Xrd3nwnWl5xThmNr5U/F0pI= @@ -2653,8 +2661,9 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 69b6a2f3c97..482b5feeb22 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-ucfg" @@ -29,7 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index ba64e489cb7..24c93d4580d 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/appender.go b/libbeat/autodiscover/appender.go index cbabb4e3441..3bb14a5ae18 100644 --- a/libbeat/autodiscover/appender.go +++ b/libbeat/autodiscover/appender.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/appender_test.go b/libbeat/autodiscover/appender_test.go index b1d4fdbcc37..8493cbfc8f0 100644 --- a/libbeat/autodiscover/appender_test.go +++ b/libbeat/autodiscover/appender_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 450fec297b6..d9dcf8c49a9 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -24,9 +24,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/appenders/config/config.go.orig b/libbeat/autodiscover/appenders/config/config.go.orig deleted file mode 100644 index ef16d4b1289..00000000000 --- a/libbeat/autodiscover/appenders/config/config.go.orig +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package config - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/autodiscover" - "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/beats/v7/libbeat/logp" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func init() { - autodiscover.Registry.AddAppender("config", NewConfigAppender) -} - -type config struct { - ConditionConfig *conditions.Config `config:"condition"` - Config *conf.C `config:"config"` -} - -type configAppender struct { - condition conditions.Condition - config mapstr.M -} - -// NewConfigAppender creates a configAppender that can append templatized configs into built configs -func NewConfigAppender(cfg *conf.C) (autodiscover.Appender, error) { - cfgwarn.Beta("The config appender is beta") - - config := config{} - err := cfg.Unpack(&config) - if err != nil { - return nil, fmt.Errorf("unable to unpack config appender due to error: %+v", err) - } - - var cond conditions.Condition - - if config.ConditionConfig != nil { - cond, err = conditions.NewCondition(config.ConditionConfig) - if err != nil { - return nil, errors.Wrap(err, "unable to create condition due to error") - } - } - - // Unpack the config - cf := mapstr.M{} - err = config.Config.Unpack(&cf) - if err != nil { - return nil, errors.Wrap(err, "unable to unpack config due to error") - } - - return &configAppender{condition: cond, config: cf}, nil -} - -// Append adds configuration into configs built by builds/templates. It applies conditions to filter out -// configs to apply, applies them and tries to apply templates if any are present. -func (c *configAppender) Append(event bus.Event) { - cfgsRaw, ok := event["config"] - // There are no configs - if !ok { - return - } - - cfgs, ok := cfgsRaw.([]*conf.C) - // Config key doesnt have an array of config objects - if !ok { - return - } - if c.condition == nil || c.condition.Check(mapstr.M(event)) == true { - // Merge the template with all the configs - for _, cfg := range cfgs { - cf := mapstr.M{} - err := cfg.Unpack(&cf) - if err != nil { - logp.Debug("config", "unable to unpack config due to error: %v", err) - continue - } - err = cfg.Merge(&c.config) - if err != nil { - logp.Debug("config", "unable to merge configs due to error: %v", err) - } - } - - // Apply the template - template.ApplyConfigTemplate(event, cfgs) - } - - // Replace old config with newly appended configs - event["config"] = cfgs -} diff --git a/libbeat/autodiscover/appenders/config/config_test.go b/libbeat/autodiscover/appenders/config/config_test.go index 45a86c62813..cbc1fd4d5c1 100644 --- a/libbeat/autodiscover/appenders/config/config_test.go +++ b/libbeat/autodiscover/appenders/config/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index 58a34ef0443..ebccfd9eb82 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -26,10 +26,10 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/meta" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/autodiscover_test.go b/libbeat/autodiscover/autodiscover_test.go index d7ba66c01ba..1cc8bbe8b6d 100644 --- a/libbeat/autodiscover/autodiscover_test.go +++ b/libbeat/autodiscover/autodiscover_test.go @@ -29,10 +29,10 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index bd3ca34751a..7cede29fcb2 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -22,11 +22,10 @@ import ( "fmt" "strings" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/go-ucfg" - - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" ) // Builder provides an interface by which configs can be built from provider metadata @@ -39,7 +38,7 @@ type Builder interface { // has access to a keystores registry type Builders struct { builders []Builder - keystoreProvider keystore.Provider + keystoreProvider bus.KeystoreProvider } // BuilderConstructor is a func used to generate a Builder object @@ -120,7 +119,7 @@ func (b Builders) GetConfig(event bus.Event) []*config.C { func NewBuilders( bConfigs []*config.C, hintsCfg *config.C, - keystoreProvider keystore.Provider, + keystoreProvider bus.KeystoreProvider, ) (Builders, error) { var builders Builders if hintsCfg.Enabled() { diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index ee7b3006844..160b516a4f6 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -22,10 +22,9 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/go-ucfg" - - "github.com/elastic/beats/v7/libbeat/common/bus" ) type fakeBuilder struct{} diff --git a/libbeat/autodiscover/eventselect.go b/libbeat/autodiscover/eventselect.go index af94e73d4cf..e7a05123be0 100644 --- a/libbeat/autodiscover/eventselect.go +++ b/libbeat/autodiscover/eventselect.go @@ -20,7 +20,7 @@ package autodiscover import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/provider.go b/libbeat/autodiscover/provider.go index c80245cc749..8d70e80ee29 100644 --- a/libbeat/autodiscover/provider.go +++ b/libbeat/autodiscover/provider.go @@ -24,9 +24,9 @@ import ( "github.com/gofrs/uuid" "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" ) // Provider for autodiscover diff --git a/libbeat/autodiscover/providers/docker/config.go b/libbeat/autodiscover/providers/docker/config.go index 2eb9f670815..507b51e14ab 100644 --- a/libbeat/autodiscover/providers/docker/config.go +++ b/libbeat/autodiscover/providers/docker/config.go @@ -24,7 +24,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 07fe0022acc..cf233202787 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -32,10 +32,10 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/docker" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/safemapstr" diff --git a/libbeat/autodiscover/providers/docker/docker_integration_test.go b/libbeat/autodiscover/providers/docker/docker_integration_test.go index 15febc6460c..284b93c1c0e 100644 --- a/libbeat/autodiscover/providers/docker/docker_integration_test.go +++ b/libbeat/autodiscover/providers/docker/docker_integration_test.go @@ -28,10 +28,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" dk "github.com/elastic/beats/v7/libbeat/tests/docker" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/docker/docker_test.go b/libbeat/autodiscover/providers/docker/docker_test.go index 4f14e63490b..309a06f0b67 100644 --- a/libbeat/autodiscover/providers/docker/docker_test.go +++ b/libbeat/autodiscover/providers/docker/docker_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/jolokia/discovery.go b/libbeat/autodiscover/providers/jolokia/discovery.go index 9c3d122020c..20fc379a91e 100644 --- a/libbeat/autodiscover/providers/jolokia/discovery.go +++ b/libbeat/autodiscover/providers/jolokia/discovery.go @@ -27,9 +27,9 @@ import ( "github.com/gofrs/uuid" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common/bus" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index d7f677f8a73..594fed24bc0 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -25,9 +25,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" ) func init() { diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index 050f271a6d0..31a807d2489 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -24,11 +24,10 @@ import ( "fmt" "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/beats/v7/libbeat/autodiscover/template" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index c5a22e23acc..710f69785bc 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -22,12 +22,12 @@ import ( "github.com/stretchr/testify/assert" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-ucfg" "github.com/elastic/beats/v7/libbeat/autodiscover" - "github.com/elastic/beats/v7/libbeat/common/bus" ) func TestConfigWithCustomBuilders(t *testing.T) { diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index 748b465d136..9f0d93ed048 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -35,11 +35,11 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/k8skeystore" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/k8skeystore" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go index 1c7411c96e9..7bdec78ce5c 100644 --- a/libbeat/autodiscover/providers/kubernetes/node.go +++ b/libbeat/autodiscover/providers/kubernetes/node.go @@ -30,9 +30,9 @@ import ( k8s "k8s.io/client-go/kubernetes" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index abc51d94866..5336ddde758 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index ff36975efc8..a4077a23aae 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -29,9 +29,9 @@ import ( k8s "k8s.io/client-go/kubernetes" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index b0a27a5c06b..9c4218bee85 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -30,9 +30,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index e49597a9ca9..38b6d5409ad 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -28,9 +28,9 @@ import ( k8s "k8s.io/client-go/kubernetes" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index f824f068a27..510ac6ebd0d 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index 1735973975b..c050ff8acd8 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -25,9 +25,9 @@ import ( "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/parse" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" ) @@ -38,7 +38,7 @@ import ( type Mapper struct { ConditionMaps []*ConditionMap keystore keystore.Keystore - keystoreProvider keystore.Provider + keystoreProvider bus.KeystoreProvider } // ConditionMap maps a condition to the configs to use when it's triggered @@ -57,7 +57,7 @@ type MapperSettings []*struct { func NewConfigMapper( configs MapperSettings, keystore keystore.Keystore, - keystoreProvider keystore.Provider, + keystoreProvider bus.KeystoreProvider, ) (mapper Mapper, err error) { for _, c := range configs { condMap := &ConditionMap{Configs: c.Configs} diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index 9e6f48ae948..d1ac8152c47 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -25,9 +25,9 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -279,7 +279,7 @@ type mockKeystore struct { secret string } -func newMockKeystoreProvider(secret string) keystore.Provider { +func newMockKeystoreProvider(secret string) bus.KeystoreProvider { return &mockKeystore{secret} } diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 0a44f7c12d0..5cf0c932ca8 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -20,9 +20,9 @@ package beat import ( "github.com/elastic/beats/v7/libbeat/common/reload" "github.com/elastic/beats/v7/libbeat/instrumentation" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" ) // Creator initializes and configures a new Beater instance used to execute diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index f54ae8b3352..fc6faef705f 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -52,7 +52,6 @@ import ( "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/idxmgmt" "github.com/elastic/beats/v7/libbeat/instrumentation" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/libbeat/kibana" "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/metric/system/host" @@ -67,6 +66,7 @@ import ( "github.com/elastic/beats/v7/libbeat/version" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/file" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/configure" "github.com/elastic/elastic-agent-libs/mapstr" @@ -1135,7 +1135,7 @@ func obfuscateConfigOpts() []ucfg.Option { func LoadKeystore(cfg *config.C, name string) (keystore.Keystore, error) { keystoreCfg, _ := cfg.Child("keystore", -1) defaultPathConfig := paths.Resolve(paths.Data, fmt.Sprintf("%s.keystore", name)) - return keystore.Factory(keystoreCfg, defaultPathConfig) + return keystore.Factory(keystoreCfg, defaultPathConfig, common.IsStrictPerms()) } func InitKibanaConfig(beatConfig beatConfig) *config.C { diff --git a/libbeat/cmd/keystore.go b/libbeat/cmd/keystore.go index 1b635d95ba2..dad7b4745a0 100644 --- a/libbeat/cmd/keystore.go +++ b/libbeat/cmd/keystore.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/cmd/instance" "github.com/elastic/beats/v7/libbeat/common/cli" "github.com/elastic/beats/v7/libbeat/common/terminal" - "github.com/elastic/beats/v7/libbeat/keystore" + "github.com/elastic/elastic-agent-libs/keystore" ) func getKeystore(settings instance.Settings) (keystore.Keystore, error) { diff --git a/libbeat/common/bus/bus.go b/libbeat/common/bus/bus.go deleted file mode 100644 index 921f2c64393..00000000000 --- a/libbeat/common/bus/bus.go +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package bus - -import ( - "sync" - - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -// Event sent to the bus -type Event mapstr.M - -// Bus provides a common channel to emit and listen for Events -type Bus interface { - // Publish an event to the bus - Publish(Event) - - // Subscribe to all events, filter them to the ones containing *all* the keys in filter - Subscribe(filter ...string) Listener -} - -// Listener retrieves Events from a Bus subscription until Stop is called -type Listener interface { - // Events channel - Events() <-chan Event - - // Stop listening and removes itself from the bus - Stop() -} - -type bus struct { - sync.RWMutex - log *logp.Logger - listeners []*listener - store chan Event -} - -type listener struct { - filter []string - channel chan Event - bus *bus -} - -// New initializes a new bus with the given name and returns it -func New(log *logp.Logger, name string) Bus { - return &bus{ - log: createLogger(log, name), - listeners: make([]*listener, 0), - } -} - -// NewBusWithStore allows to create a buffered bus when producers send data without -// listeners being subscribed to them. size determines the size of the buffer. -func NewBusWithStore(log *logp.Logger, name string, size int) Bus { - return &bus{ - log: createLogger(log, name), - listeners: make([]*listener, 0), - store: make(chan Event, size), - } -} - -func createLogger(log *logp.Logger, name string) *logp.Logger { - selector := "bus-" + name - return log.Named(selector).With("libbeat.bus", name) -} - -func (b *bus) Publish(e Event) { - b.RLock() - defer b.RUnlock() - - b.log.Debugf("%+v", e) - if len(b.listeners) == 0 && b.store != nil { - b.store <- e - return - } - - if b.store != nil && len(b.store) != 0 { - doBreak := false - for !doBreak { - select { - case eve := <-b.store: - for _, listener := range b.listeners { - if listener.interested(eve) { - listener.channel <- eve - } - } - default: - doBreak = true - } - } - } - - for _, listener := range b.listeners { - if listener.interested(e) { - listener.channel <- e - } - } -} - -func (b *bus) Subscribe(filter ...string) Listener { - listener := &listener{ - filter: filter, - bus: b, - channel: make(chan Event, 100), - } - - b.Lock() - defer b.Unlock() - b.listeners = append(b.listeners, listener) - - return listener -} - -func (l *listener) Events() <-chan Event { - return l.channel -} - -func (l *listener) Stop() { - l.bus.Lock() - defer l.bus.Unlock() - - for i, listener := range l.bus.listeners { - if l == listener { - l.bus.listeners = append(l.bus.listeners[:i], l.bus.listeners[i+1:]...) - } - } - - close(l.channel) -} - -// Return true if listener is interested on the given event -func (l *listener) interested(e Event) bool { - for _, key := range l.filter { - if _, ok := e[key]; !ok { - return false - } - } - return true -} diff --git a/libbeat/common/bus/bus_test.go b/libbeat/common/bus/bus_test.go deleted file mode 100644 index dcde9717267..00000000000 --- a/libbeat/common/bus/bus_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package bus - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-libs/logp" -) - -func TestEmit(t *testing.T) { - bus := New(logp.L(), "name") - listener := bus.Subscribe() - - bus.Publish(Event{ - "foo": "bar", - }) - - event := <-listener.Events() - assert.Equal(t, event["foo"], "bar") -} - -func TestEmitOrder(t *testing.T) { - bus := New(logp.L(), "name") - listener := bus.Subscribe() - bus.Publish(Event{"first": "event"}) - bus.Publish(Event{"second": "event"}) - - event1 := <-listener.Events() - event2 := <-listener.Events() - assert.Equal(t, event1, Event{"first": "event"}) - assert.Equal(t, event2, Event{"second": "event"}) -} - -func TestSubscribeFilter(t *testing.T) { - bus := New(logp.L(), "name") - listener := bus.Subscribe("second") - - bus.Publish(Event{"first": "event"}) - bus.Publish(Event{"second": "event"}) - - event := <-listener.Events() - assert.Equal(t, event, Event{"second": "event"}) -} - -func TestMultipleListeners(t *testing.T) { - bus := New(logp.L(), "name") - listener1 := bus.Subscribe("a") - listener2 := bus.Subscribe("a", "b") - - bus.Publish(Event{"a": "event"}) - bus.Publish(Event{"a": 1, "b": 2}) - - event1 := <-listener1.Events() - event2 := <-listener1.Events() - assert.Equal(t, event1, Event{"a": "event"}) - assert.Equal(t, event2, Event{"a": 1, "b": 2}) - - event1 = <-listener2.Events() - assert.Equal(t, event1, Event{"a": 1, "b": 2}) - - select { - case event2 = <-listener2.Events(): - t.Error("Got unexpected event:", event2) - default: - } -} - -func TestListenerClose(t *testing.T) { - bus := New(logp.L(), "name") - listener := bus.Subscribe() - - bus.Publish(Event{"first": "event"}) - bus.Publish(Event{"second": "event"}) - - listener.Stop() - - bus.Publish(Event{"third": "event"}) - - event := <-listener.Events() - assert.Equal(t, event, Event{"first": "event"}) - event = <-listener.Events() - assert.Equal(t, event, Event{"second": "event"}) - - // Channel was closed, we get an empty event - event = <-listener.Events() - assert.Equal(t, event, Event(nil)) -} - -func TestUnsubscribedBus(t *testing.T) { - bus := NewBusWithStore(logp.L(), "name", 2) - bus.Publish(Event{"first": "event"}) - - listener := bus.Subscribe() - bus.Publish(Event{"second": "event"}) - event := <-listener.Events() - event1 := <-listener.Events() - assert.Equal(t, event, Event{"first": "event"}) - assert.Equal(t, event1, Event{"second": "event"}) - - bus.Publish(Event{"a": 1, "b": 2}) - event2 := <-listener.Events() - assert.Equal(t, event2, Event{"a": 1, "b": 2}) -} diff --git a/libbeat/common/docker/client.go b/libbeat/common/docker/client.go deleted file mode 100644 index c435f46c2e6..00000000000 --- a/libbeat/common/docker/client.go +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux || darwin || windows -// +build linux darwin windows - -package docker - -import ( - "net/http" - "os" - - "github.com/docker/docker/client" - - "github.com/elastic/elastic-agent-libs/logp" -) - -// NewClient builds and returns a new Docker client. On the first request the -// client will negotiate the API version with the server unless -// DOCKER_API_VERSION is set in the environment. -func NewClient(host string, httpClient *http.Client, httpHeaders map[string]string) (*client.Client, error) { - log := logp.NewLogger("docker") - - opts := []client.Opt{ - client.WithHost(host), - client.WithHTTPClient(httpClient), - client.WithHTTPHeaders(httpHeaders), - } - - version := os.Getenv("DOCKER_API_VERSION") - if version != "" { - log.Debugf("Docker client will use API version %v as set by the DOCKER_API_VERSION environment variable.", version) - opts = append(opts, client.WithVersion(version)) - } else { - log.Debug("Docker client will negotiate the API version on the first request.") - opts = append(opts, client.WithAPIVersionNegotiation()) - } - - return client.NewClientWithOpts(opts...) -} diff --git a/libbeat/common/docker/client_test.go b/libbeat/common/docker/client_test.go deleted file mode 100644 index af6f7c6d089..00000000000 --- a/libbeat/common/docker/client_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build integration && (linux || darwin || windows) -// +build integration -// +build linux darwin windows - -package docker - -import ( - "os" - "testing" - - "github.com/docker/docker/api" - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/stretchr/testify/assert" - "golang.org/x/net/context" -) - -func TestNewClient(t *testing.T) { - c, err := NewClient(client.DefaultDockerHost, nil, nil) - assert.NoError(t, err) - assert.NotNil(t, c) - - _, err = c.ContainerList(context.Background(), types.ContainerListOptions{}) - assert.NoError(t, err) - - // This test only works on newer Docker versions (any supported one really) - switch c.ClientVersion() { - case "1.22": - t.Skip("Docker version is too old for this test") - case api.DefaultVersion: - t.Logf("Using default API version: %s", api.DefaultVersion) - default: - t.Logf("Negotiated version: %s", c.ClientVersion()) - } - - // Test we can hardcode version - os.Setenv("DOCKER_API_VERSION", "1.22") - - c, err = NewClient(client.DefaultDockerHost, nil, nil) - assert.NoError(t, err) - assert.NotNil(t, c) - assert.Equal(t, "1.22", c.ClientVersion()) - - _, err = c.ContainerList(context.Background(), types.ContainerListOptions{}) - assert.NoError(t, err) -} diff --git a/libbeat/common/docker/helpers.go b/libbeat/common/docker/helpers.go deleted file mode 100644 index 74cfd2f923d..00000000000 --- a/libbeat/common/docker/helpers.go +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package docker - -import ( - "strings" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/safemapstr" -) - -// ExtractContainerName strips the `/` characters that frequently appear in container names -func ExtractContainerName(names []string) string { - output := names[0] - - if len(names) > 1 { - for _, name := range names { - if strings.Count(output, "/") > strings.Count(name, "/") { - output = name - } - } - } - return strings.Trim(output, "/") -} - -// DeDotLabels returns a new mapstr.M containing a copy of the labels -// where the dots have been converted into nested structure, avoiding -// possible mapping errors -func DeDotLabels(labels map[string]string, dedot bool) mapstr.M { - outputLabels := mapstr.M{} - for k, v := range labels { - if dedot { - // This is necessary so that ES does not interpret '.' fields as new - // nested JSON objects, and also makes this compatible with ES 2.x. - label := common.DeDot(k) - outputLabels.Put(label, v) - } else { - // If we don't dedot we ensure there are no mapping errors with safemapstr - safemapstr.Put(outputLabels, k, v) - } - } - - return outputLabels -} diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go deleted file mode 100644 index 0f18e15ea31..00000000000 --- a/libbeat/common/docker/watcher.go +++ /dev/null @@ -1,480 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux || darwin || windows -// +build linux darwin windows - -package docker - -import ( - "context" - "io" - "net/http" - "sync" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/events" - "github.com/docker/docker/api/types/filters" - "github.com/docker/go-connections/tlsconfig" - - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/elastic-agent-libs/logp" -) - -// Select Docker API version -const ( - shortIDLen = 12 - dockerRequestTimeout = 10 * time.Second - dockerEventsWatchPityTimerInterval = 10 * time.Second - dockerEventsWatchPityTimerTimeout = 10 * time.Minute -) - -// Watcher reads docker events and keeps a list of known containers -type Watcher interface { - // Start watching docker API for new containers - Start() error - - // Stop watching docker API for new containers - Stop() - - // Container returns the running container with the given ID or nil if unknown - Container(ID string) *Container - - // Containers returns the list of known containers - Containers() map[string]*Container - - // ListenStart returns a bus listener to receive container started events, with a `container` key holding it - ListenStart() bus.Listener - - // ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it - ListenStop() bus.Listener -} - -// TLSConfig for docker socket connection -type TLSConfig struct { - CA string `config:"certificate_authority"` - Certificate string `config:"certificate"` - Key string `config:"key"` -} - -type watcher struct { - sync.RWMutex - log *logp.Logger - client Client - ctx context.Context - stop context.CancelFunc - containers map[string]*Container - deleted map[string]time.Time // deleted annotations key -> last access time - cleanupTimeout time.Duration - clock clock - stopped sync.WaitGroup - bus bus.Bus - shortID bool // whether to store short ID in "containers" too -} - -// clock is an interface used to provide mocked time on testing -type clock interface { - Now() time.Time -} - -// systemClock implements the clock interface using the system clock via the time package -type systemClock struct{} - -// Now returns the current time -func (*systemClock) Now() time.Time { return time.Now() } - -// Container info retrieved by the watcher -type Container struct { - ID string - Name string - Image string - Labels map[string]string - IPAddresses []string - Ports []types.Port -} - -// Client for docker interface -type Client interface { - ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) - ContainerInspect(ctx context.Context, container string) (types.ContainerJSON, error) - Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error) -} - -// WatcherConstructor represent a function that creates a new Watcher from giving parameters -type WatcherConstructor func(logp *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) - -// NewWatcher returns a watcher running for the given settings -func NewWatcher(log *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) { - var httpClient *http.Client - if tls != nil { - options := tlsconfig.Options{ - CAFile: tls.CA, - CertFile: tls.Certificate, - KeyFile: tls.Key, - } - - tlsc, err := tlsconfig.Client(options) - if err != nil { - return nil, err - } - - httpClient = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsc, - }, - } - } - - client, err := NewClient(host, httpClient, nil) - if err != nil { - return nil, err - } - - // Extra check to confirm that Docker is available - _, err = client.Info(context.Background()) - if err != nil { - client.Close() - return nil, err - } - - return NewWatcherWithClient(log, client, 60*time.Second, storeShortID) -} - -// NewWatcherWithClient creates a new Watcher from a given Docker client -func NewWatcherWithClient(log *logp.Logger, client Client, cleanupTimeout time.Duration, storeShortID bool) (Watcher, error) { - ctx, cancel := context.WithCancel(context.Background()) - return &watcher{ - log: log, - client: client, - ctx: ctx, - stop: cancel, - containers: make(map[string]*Container), - deleted: make(map[string]time.Time), - cleanupTimeout: cleanupTimeout, - bus: bus.New(log, "docker"), - shortID: storeShortID, - clock: &systemClock{}, - }, nil -} - -// Container returns the running container with the given ID or nil if unknown -func (w *watcher) Container(ID string) *Container { - w.RLock() - container := w.containers[ID] - if container == nil { - w.RUnlock() - return nil - } - _, ok := w.deleted[container.ID] - w.RUnlock() - - // Update last access time if it's deleted - if ok { - w.Lock() - w.deleted[container.ID] = w.clock.Now() - w.Unlock() - } - - return container -} - -// Containers returns the list of known containers -func (w *watcher) Containers() map[string]*Container { - w.RLock() - defer w.RUnlock() - res := make(map[string]*Container) - for k, v := range w.containers { - if !w.shortID || len(k) != shortIDLen { - res[k] = v - } - } - return res -} - -// Start watching docker API for new containers -func (w *watcher) Start() error { - // Do initial scan of existing containers - w.log.Debug("Start docker containers scanner") - - w.Lock() - defer w.Unlock() - containers, err := w.listContainers(types.ContainerListOptions{}) - if err != nil { - return err - } - - for _, c := range containers { - w.containers[c.ID] = c - if w.shortID { - w.containers[c.ID[:shortIDLen]] = c - } - } - - // Emit all start events (avoid blocking if the bus get's blocked) - go func() { - for _, c := range containers { - w.bus.Publish(bus.Event{ - "start": true, - "container": c, - }) - } - }() - - w.stopped.Add(2) - go w.watch() - go w.cleanupWorker() - - return nil -} - -func (w *watcher) Stop() { - w.stop() - w.stopped.Wait() -} - -func (w *watcher) watch() { - defer w.stopped.Done() - - filter := filters.NewArgs() - filter.Add("type", "container") - - // Ticker to restart the watcher when no events are received after some time. - tickChan := time.NewTicker(dockerEventsWatchPityTimerInterval) - defer tickChan.Stop() - - lastValidTimestamp := w.clock.Now() - - watch := func() bool { - lastReceivedEventTime := w.clock.Now() - - w.log.Debugf("Fetching events since %s", lastValidTimestamp) - - options := types.EventsOptions{ - Since: lastValidTimestamp.Format(time.RFC3339Nano), - Filters: filter, - } - - ctx, cancel := context.WithCancel(w.ctx) - defer cancel() - - events, errors := w.client.Events(ctx, options) - for { - select { - case event := <-events: - w.log.Debugf("Got a new docker event: %v", event) - if event.TimeNano > 0 { - lastValidTimestamp = time.Unix(0, event.TimeNano) - } else { - lastValidTimestamp = time.Unix(event.Time, 0) - } - lastReceivedEventTime = w.clock.Now() - - switch event.Action { - case "start", "update": - w.containerUpdate(event) - case "die": - w.containerDelete(event) - } - case err := <-errors: - switch err { - case io.EOF: - // Client disconnected, watch is not done, reconnect - w.log.Debug("EOF received in events stream, restarting watch call") - case context.DeadlineExceeded: - w.log.Debug("Context deadline exceeded for docker request, restarting watch call") - case context.Canceled: - // Parent context has been canceled, watch is done. - return true - default: - w.log.Errorf("Error watching for docker events: %+v", err) - } - return false - case <-tickChan.C: - if time.Since(lastReceivedEventTime) > dockerEventsWatchPityTimerTimeout { - w.log.Infof("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout) - return false - } - case <-w.ctx.Done(): - w.log.Debug("Watcher stopped") - return true - } - } - } - - for { - done := watch() - if done { - return - } - // Wait before trying to reconnect - time.Sleep(1 * time.Second) - } -} - -func (w *watcher) containerUpdate(event events.Message) { - filter := filters.NewArgs() - filter.Add("id", event.Actor.ID) - - containers, err := w.listContainers(types.ContainerListOptions{ - Filters: filter, - }) - if err != nil || len(containers) != 1 { - w.log.Errorf("Error getting container info: %v", err) - return - } - container := containers[0] - - w.Lock() - w.containers[event.Actor.ID] = container - if w.shortID { - w.containers[event.Actor.ID[:shortIDLen]] = container - } - // un-delete if it's flagged (in case of update or recreation) - delete(w.deleted, event.Actor.ID) - w.Unlock() - - w.bus.Publish(bus.Event{ - "start": true, - "container": container, - }) -} - -func (w *watcher) containerDelete(event events.Message) { - container := w.Container(event.Actor.ID) - - w.Lock() - w.deleted[event.Actor.ID] = w.clock.Now() - w.Unlock() - - if container != nil { - w.bus.Publish(bus.Event{ - "stop": true, - "container": container, - }) - } -} - -func (w *watcher) listContainers(options types.ContainerListOptions) ([]*Container, error) { - log := w.log - - log.Debug("List containers") - ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) - defer cancel() - - containers, err := w.client.ContainerList(ctx, options) - if err != nil { - return nil, err - } - - var result []*Container - for _, c := range containers { - var ipaddresses []string - if c.NetworkSettings != nil { - // Handle alternate platforms like VMWare's VIC that might not have this data. - for _, net := range c.NetworkSettings.Networks { - if net.IPAddress != "" { - ipaddresses = append(ipaddresses, net.IPAddress) - } - } - } - - // If there are no network interfaces, assume that the container is on host network - // Inspect the container directly and use the hostname as the IP address in order - if len(ipaddresses) == 0 { - log.Debugf("Inspect container %s", c.ID) - ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) - defer cancel() - info, err := w.client.ContainerInspect(ctx, c.ID) - if err == nil { - ipaddresses = append(ipaddresses, info.Config.Hostname) - } else { - log.Warnf("unable to inspect container %s due to error %+v", c.ID, err) - } - } - result = append(result, &Container{ - ID: c.ID, - Name: c.Names[0][1:], // Strip '/' from container names - Image: c.Image, - Labels: c.Labels, - Ports: c.Ports, - IPAddresses: ipaddresses, - }) - } - - return result, nil -} - -// Clean up deleted containers after they are not used anymore -func (w *watcher) cleanupWorker() { - defer w.stopped.Done() - - for { - select { - case <-w.ctx.Done(): - return - // Wait a full period - case <-time.After(w.cleanupTimeout): - w.runCleanup() - } - } -} - -func (w *watcher) runCleanup() { - // Check entries for timeout - var toDelete []string - timeout := w.clock.Now().Add(-w.cleanupTimeout) - w.RLock() - for key, lastSeen := range w.deleted { - if lastSeen.Before(timeout) { - w.log.Debugf("Removing container %s after cool down timeout", key) - toDelete = append(toDelete, key) - } - } - w.RUnlock() - - // Delete timed out entries: - for _, key := range toDelete { - container := w.Container(key) - if container != nil { - w.bus.Publish(bus.Event{ - "delete": true, - "container": container, - }) - } - } - - w.Lock() - for _, key := range toDelete { - delete(w.deleted, key) - delete(w.containers, key) - if w.shortID { - delete(w.containers, key[:shortIDLen]) - } - } - w.Unlock() -} - -// ListenStart returns a bus listener to receive container started events, with a `container` key holding it -func (w *watcher) ListenStart() bus.Listener { - return w.bus.Subscribe("start") -} - -// ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it -func (w *watcher) ListenStop() bus.Listener { - return w.bus.Subscribe("stop") -} diff --git a/libbeat/common/docker/watcher_test.go b/libbeat/common/docker/watcher_test.go deleted file mode 100644 index ddebef5b575..00000000000 --- a/libbeat/common/docker/watcher_test.go +++ /dev/null @@ -1,523 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build linux || darwin || windows -// +build linux darwin windows - -package docker - -import ( - "errors" - "sync" - "testing" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/events" - "github.com/stretchr/testify/assert" - "golang.org/x/net/context" - - "github.com/elastic/elastic-agent-libs/logp" -) - -type MockClient struct { - // containers to return on ContainerList call - containers [][]types.Container - // event list to send on Events call - events []interface{} - // done channel is closed when the client has sent all events - done chan interface{} -} - -func (m *MockClient) ContainerList(ctx context.Context, options types.ContainerListOptions) ([]types.Container, error) { - res := m.containers[0] - m.containers = m.containers[1:] - return res, nil -} - -func (m *MockClient) Events(ctx context.Context, options types.EventsOptions) (<-chan events.Message, <-chan error) { - eventsC := make(chan events.Message) - errorsC := make(chan error) - - go func() { - for _, event := range m.events { - switch e := event.(type) { - case events.Message: - eventsC <- e - case error: - errorsC <- e - } - } - close(m.done) - }() - - return eventsC, errorsC -} - -func (m *MockClient) ContainerInspect(ctx context.Context, container string) (types.ContainerJSON, error) { - return types.ContainerJSON{}, errors.New("unimplemented") -} - -func TestWatcherInitialization(t *testing.T) { - watcher := runAndWait(testWatcher(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "0332dbd79e20", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - types.Container{ - ID: "6ac6ee8df5d4", - Names: []string{"/other"}, - Image: "nginx", - Labels: map[string]string{}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - nil, - )) - - assert.Equal(t, map[string]*Container{ - "0332dbd79e20": &Container{ - ID: "0332dbd79e20", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - }, - "6ac6ee8df5d4": &Container{ - ID: "6ac6ee8df5d4", - Name: "other", - Image: "nginx", - Labels: map[string]string{}, - }, - }, watcher.Containers()) -} - -func TestWatcherInitializationShortID(t *testing.T) { - watcher := runAndWait(testWatcherShortID(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "1234567890123", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - types.Container{ - ID: "2345678901234", - Names: []string{"/other"}, - Image: "nginx", - Labels: map[string]string{}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - nil, - true, - )) - - assert.Equal(t, map[string]*Container{ - "1234567890123": &Container{ - ID: "1234567890123", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - }, - "2345678901234": &Container{ - ID: "2345678901234", - Name: "other", - Image: "nginx", - Labels: map[string]string{}, - }, - }, watcher.Containers()) - - assert.Equal(t, &Container{ - ID: "1234567890123", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - }, watcher.Container("123456789012")) -} - -func TestWatcherAddEvents(t *testing.T) { - watcher := runAndWait(testWatcher(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "0332dbd79e20", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - []types.Container{ - types.Container{ - ID: "6ac6ee8df5d4", - Names: []string{"/other"}, - Image: "nginx", - Labels: map[string]string{"label": "value"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "start", - Actor: events.Actor{ - ID: "6ac6ee8df5d4", - Attributes: map[string]string{ - "name": "other", - "image": "nginx", - "label": "value", - }, - }, - }, - }, - )) - - assert.Equal(t, map[string]*Container{ - "0332dbd79e20": &Container{ - ID: "0332dbd79e20", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - }, - "6ac6ee8df5d4": &Container{ - ID: "6ac6ee8df5d4", - Name: "other", - Image: "nginx", - Labels: map[string]string{"label": "value"}, - }, - }, watcher.Containers()) -} - -func TestWatcherAddEventsShortID(t *testing.T) { - watcher := runAndWait(testWatcherShortID(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "1234567890123", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - []types.Container{ - types.Container{ - ID: "2345678901234", - Names: []string{"/other"}, - Image: "nginx", - Labels: map[string]string{"label": "value"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "start", - Actor: events.Actor{ - ID: "2345678901234", - Attributes: map[string]string{ - "name": "other", - "image": "nginx", - "label": "value", - }, - }, - }, - }, - true, - )) - - assert.Equal(t, map[string]*Container{ - "1234567890123": &Container{ - ID: "1234567890123", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"foo": "bar"}, - }, - "2345678901234": &Container{ - ID: "2345678901234", - Name: "other", - Image: "nginx", - Labels: map[string]string{"label": "value"}, - }, - }, watcher.Containers()) -} - -func TestWatcherUpdateEvent(t *testing.T) { - watcher := runAndWait(testWatcher(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "0332dbd79e20", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "foo"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - []types.Container{ - types.Container{ - ID: "0332dbd79e20", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "update", - Actor: events.Actor{ - ID: "0332dbd79e20", - Attributes: map[string]string{ - "name": "containername", - "image": "busybox", - "label": "bar", - }, - }, - }, - }, - )) - - assert.Equal(t, map[string]*Container{ - "0332dbd79e20": &Container{ - ID: "0332dbd79e20", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"label": "bar"}, - }, - }, watcher.Containers()) - assert.Equal(t, 0, len(watcher.deleted)) -} - -func TestWatcherUpdateEventShortID(t *testing.T) { - watcher := runAndWait(testWatcherShortID(t, true, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "1234567890123", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "foo"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - []types.Container{ - types.Container{ - ID: "1234567890123", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "bar"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "update", - Actor: events.Actor{ - ID: "1234567890123", - Attributes: map[string]string{ - "name": "containername", - "image": "busybox", - "label": "bar", - }, - }, - }, - }, - true, - )) - - assert.Equal(t, map[string]*Container{ - "1234567890123": &Container{ - ID: "1234567890123", - Name: "containername", - Image: "busybox", - Labels: map[string]string{"label": "bar"}, - }, - }, watcher.Containers()) - assert.Equal(t, 0, len(watcher.deleted)) -} - -func TestWatcherDie(t *testing.T) { - watcher, clientDone := testWatcher(t, false, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "0332dbd79e20", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "foo"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "die", - Actor: events.Actor{ - ID: "0332dbd79e20", - }, - }, - }, - ) - - clock := newTestClock() - watcher.clock = clock - - stopListener := watcher.ListenStop() - - watcher.Start() - defer watcher.Stop() - - // Check it doesn't get removed while we request meta for the container - for i := 0; i < 18; i++ { - watcher.Container("0332dbd79e20") - clock.Sleep(watcher.cleanupTimeout / 2) - watcher.runCleanup() - if !assert.Equal(t, 1, len(watcher.Containers())) { - break - } - } - - // Wait to be sure that the delete event has been processed - <-clientDone - <-stopListener.Events() - - // Check that after the cleanup period the container is removed - clock.Sleep(watcher.cleanupTimeout + 1*time.Second) - watcher.runCleanup() - assert.Equal(t, 0, len(watcher.Containers())) -} - -func TestWatcherDieShortID(t *testing.T) { - watcher, clientDone := testWatcherShortID(t, false, - [][]types.Container{ - []types.Container{ - types.Container{ - ID: "0332dbd79e20aaa", - Names: []string{"/containername", "othername"}, - Image: "busybox", - Labels: map[string]string{"label": "foo"}, - NetworkSettings: &types.SummaryNetworkSettings{}, - }, - }, - }, - []interface{}{ - events.Message{ - Action: "die", - Actor: events.Actor{ - ID: "0332dbd79e20aaa", - }, - }, - }, - true, - ) - - clock := newTestClock() - watcher.clock = clock - - stopListener := watcher.ListenStop() - - watcher.Start() - defer watcher.Stop() - - // Check it doesn't get removed while we request meta for the container - for i := 0; i < 18; i++ { - watcher.Container("0332dbd79e20") - clock.Sleep(watcher.cleanupTimeout / 2) - watcher.runCleanup() - if !assert.Equal(t, 1, len(watcher.Containers())) { - break - } - } - - // Wait to be sure that the delete event has been processed - <-clientDone - <-stopListener.Events() - - // Check that after the cleanup period the container is removed - clock.Sleep(watcher.cleanupTimeout + 1*time.Second) - watcher.runCleanup() - assert.Equal(t, 0, len(watcher.Containers())) -} - -func testWatcher(t *testing.T, kill bool, containers [][]types.Container, events []interface{}) (*watcher, chan interface{}) { - return testWatcherShortID(t, kill, containers, events, false) -} - -func testWatcherShortID(t *testing.T, kill bool, containers [][]types.Container, events []interface{}, enable bool) (*watcher, chan interface{}) { - logp.TestingSetup() - - client := &MockClient{ - containers: containers, - events: events, - done: make(chan interface{}), - } - - w, err := NewWatcherWithClient(logp.L(), client, 200*time.Millisecond, enable) - if err != nil { - t.Fatal(err) - } - watcher, ok := w.(*watcher) - if !ok { - t.Fatal("'watcher' was supposed to be pointer to the watcher structure") - } - - return watcher, client.done -} - -func runAndWait(w *watcher, done chan interface{}) *watcher { - w.Start() - <-done - w.Stop() - return w -} - -type testClock struct { - sync.Mutex - - now time.Time -} - -func newTestClock() *testClock { - return &testClock{now: time.Time{}} -} - -func (c *testClock) Now() time.Time { - c.Lock() - defer c.Unlock() - - c.now = c.now.Add(1) - return c.now -} - -func (c *testClock) Sleep(d time.Duration) { - c.Lock() - defer c.Unlock() - - c.now = c.now.Add(d) -} diff --git a/libbeat/common/kubernetes/eventhandler.go b/libbeat/common/kubernetes/eventhandler.go deleted file mode 100644 index b499b409fe3..00000000000 --- a/libbeat/common/kubernetes/eventhandler.go +++ /dev/null @@ -1,231 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "sync" -) - -// ResourceEventHandler can handle notifications for events that happen to a -// resource. The events are informational only, so you can't return an -// error. -// * OnAdd is called when an object is added. -// * OnUpdate is called when an object is modified. Note that oldObj is the -// last known state of the object-- it is possible that several changes -// were combined together, so you can't use this to see every single -// change. OnUpdate is also called when a re-list happens, and it will -// get called even if nothing changed. This is useful for periodically -// evaluating or syncing something. -// * OnDelete will get the final state of the item if it is known, otherwise -// it will get an object of type DeletedFinalStateUnknown. This can -// happen if the watch is closed and misses the delete event and we don't -// notice the deletion until the subsequent re-list. -// TODO: allow the On* methods to return an error so that the RateLimited WorkQueue -// TODO: can requeue the failed event processing. -type ResourceEventHandler interface { - OnAdd(obj interface{}) - OnUpdate(obj interface{}) - OnDelete(obj interface{}) -} - -// ResourceEventHandlerFuncs is an adaptor to let you easily specify as many or -// as few of the notification functions as you want while still implementing -// ResourceEventHandler. -type ResourceEventHandlerFuncs struct { - AddFunc func(obj interface{}) - UpdateFunc func(obj interface{}) - DeleteFunc func(obj interface{}) -} - -// OnAdd calls AddFunc if it's not nil. -func (r ResourceEventHandlerFuncs) OnAdd(obj interface{}) { - if r.AddFunc != nil { - r.AddFunc(obj) - } -} - -// OnUpdate calls UpdateFunc if it's not nil. -func (r ResourceEventHandlerFuncs) OnUpdate(obj interface{}) { - if r.UpdateFunc != nil { - r.UpdateFunc(obj) - } -} - -// OnDelete calls DeleteFunc if it's not nil. -func (r ResourceEventHandlerFuncs) OnDelete(obj interface{}) { - if r.DeleteFunc != nil { - r.DeleteFunc(obj) - } -} - -// NoOpEventHandlerFuncs ensures that watcher reconciliation can happen even without the required funcs -type NoOpEventHandlerFuncs struct { -} - -// OnAdd does a no-op on an add event -func (n NoOpEventHandlerFuncs) OnAdd(obj interface{}) { - -} - -// OnUpdate does a no-op on an update event -func (n NoOpEventHandlerFuncs) OnUpdate(obj interface{}) { - -} - -// OnDelete does a no-op on a delete event -func (n NoOpEventHandlerFuncs) OnDelete(obj interface{}) { - -} - -// FilteringResourceEventHandler applies the provided filter to all events coming -// in, ensuring the appropriate nested handler method is invoked. An object -// that starts passing the filter after an update is considered an add, and an -// object that stops passing the filter after an update is considered a delete. -type FilteringResourceEventHandler struct { - FilterFunc func(obj interface{}) bool - Handler ResourceEventHandler -} - -// OnAdd calls the nested handler only if the filter succeeds -func (r FilteringResourceEventHandler) OnAdd(obj interface{}) { - if !r.FilterFunc(obj) { - return - } - r.Handler.OnAdd(obj) -} - -// OnUpdate ensures the proper handler is called depending on whether the filter matches -func (r FilteringResourceEventHandler) OnUpdate(obj interface{}) { - if !r.FilterFunc(obj) { - return - } - r.Handler.OnUpdate(obj) -} - -// OnDelete calls the nested handler only if the filter succeeds -func (r FilteringResourceEventHandler) OnDelete(obj interface{}) { - if !r.FilterFunc(obj) { - return - } - r.Handler.OnDelete(obj) -} - -// podUpdaterHandlerFunc is a function that handles pod updater notifications. -type podUpdaterHandlerFunc func(interface{}) - -// podUpdaterStore is the interface that an object needs to implement to be -// used as a pod updater store. -type podUpdaterStore interface { - List() []interface{} -} - -// namespacePodUpdater notifies updates on pods when their namespaces are updated. -type namespacePodUpdater struct { - handler podUpdaterHandlerFunc - store podUpdaterStore - locker sync.Locker -} - -// NewNamespacePodUpdater creates a namespacePodUpdater -func NewNamespacePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, locker sync.Locker) *namespacePodUpdater { - return &namespacePodUpdater{ - handler: handler, - store: store, - locker: locker, - } -} - -// OnUpdate handles update events on namespaces. -func (n *namespacePodUpdater) OnUpdate(obj interface{}) { - ns, ok := obj.(*Namespace) - if !ok { - return - } - - // n.store.List() returns a snapshot at this point. If a delete is received - // from the main watcher, this loop may generate an update event after the - // delete is processed, leaving configurations that would never be deleted. - // Also this loop can miss updates, what could leave outdated configurations. - // Avoid these issues by locking the processing of events from the main watcher. - if n.locker != nil { - n.locker.Lock() - defer n.locker.Unlock() - } - for _, pod := range n.store.List() { - pod, ok := pod.(*Pod) - if ok && pod.Namespace == ns.Name { - n.handler(pod) - } - } -} - -// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this -// namespace they will generate their own add events. -func (*namespacePodUpdater) OnAdd(interface{}) {} - -// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this -// namespace they will generate their own delete events. -func (*namespacePodUpdater) OnDelete(interface{}) {} - -// nodePodUpdater notifies updates on pods when their nodes are updated. -type nodePodUpdater struct { - handler podUpdaterHandlerFunc - store podUpdaterStore - locker sync.Locker -} - -// NewNodePodUpdater creates a nodePodUpdater -func NewNodePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, locker sync.Locker) *nodePodUpdater { - return &nodePodUpdater{ - handler: handler, - store: store, - locker: locker, - } -} - -// OnUpdate handles update events on nodes. -func (n *nodePodUpdater) OnUpdate(obj interface{}) { - node, ok := obj.(*Node) - if !ok { - return - } - - // n.store.List() returns a snapshot at this point. If a delete is received - // from the main watcher, this loop may generate an update event after the - // delete is processed, leaving configurations that would never be deleted. - // Also this loop can miss updates, what could leave outdated configurations. - // Avoid these issues by locking the processing of events from the main watcher. - if n.locker != nil { - n.locker.Lock() - defer n.locker.Unlock() - } - for _, pod := range n.store.List() { - pod, ok := pod.(*Pod) - if ok && pod.Spec.NodeName == node.Name { - n.handler(pod) - } - } -} - -// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this -// namespace they will generate their own add events. -func (*nodePodUpdater) OnAdd(interface{}) {} - -// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this -// namespace they will generate their own delete events. -func (*nodePodUpdater) OnDelete(interface{}) {} diff --git a/libbeat/common/kubernetes/informer.go b/libbeat/common/kubernetes/informer.go deleted file mode 100644 index 1801a5e47ea..00000000000 --- a/libbeat/common/kubernetes/informer.go +++ /dev/null @@ -1,310 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "context" - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" -) - -func nodeSelector(options *metav1.ListOptions, opt WatchOptions) { - if opt.Node != "" { - options.FieldSelector = "spec.nodeName=" + opt.Node - } -} - -func nameSelector(options *metav1.ListOptions, name string) { - if name != "" { - options.FieldSelector = "metadata.name=" + name - } -} - -// NewInformer creates an informer for a given resource -func NewInformer(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers) (cache.SharedInformer, string, error) { - var objType string - - var listwatch *cache.ListWatch - ctx := context.TODO() - switch resource.(type) { - case *Pod: - p := client.CoreV1().Pods(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - nodeSelector(&options, opts) - return p.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - nodeSelector(&options, opts) - return p.Watch(ctx, options) - }, - } - - objType = "pod" - case *Event: - e := client.CoreV1().Events(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return e.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return e.Watch(ctx, options) - }, - } - - objType = "event" - case *Node: - n := client.CoreV1().Nodes() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - nameSelector(&options, opts.Node) - return n.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - nameSelector(&options, opts.Node) - return n.Watch(ctx, options) - }, - } - - objType = "node" - case *Namespace: - ns := client.CoreV1().Namespaces() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - nameSelector(&options, opts.Namespace) - return ns.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - nameSelector(&options, opts.Namespace) - return ns.Watch(ctx, options) - }, - } - - objType = "namespace" - case *Deployment: - d := client.AppsV1().Deployments(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return d.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return d.Watch(ctx, options) - }, - } - - objType = "deployment" - case *ReplicaSet: - rs := client.AppsV1().ReplicaSets(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return rs.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return rs.Watch(ctx, options) - }, - } - - objType = "replicaset" - case *StatefulSet: - ss := client.AppsV1().StatefulSets(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return ss.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return ss.Watch(ctx, options) - }, - } - - objType = "statefulset" - case *DaemonSet: - ss := client.AppsV1().DaemonSets(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return ss.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return ss.Watch(ctx, options) - }, - } - - objType = "daemonset" - case *Service: - svc := client.CoreV1().Services(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return svc.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return svc.Watch(ctx, options) - }, - } - - objType = "service" - case *ServiceAccount: - sa := client.CoreV1().ServiceAccounts(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return sa.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return sa.Watch(ctx, options) - }, - } - - objType = "serviceAccount" - case *CronJob: - cronjob := client.BatchV1().CronJobs(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return cronjob.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return cronjob.Watch(ctx, options) - }, - } - - objType = "cronjob" - case *Job: - job := client.BatchV1().Jobs(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return job.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return job.Watch(ctx, options) - }, - } - - objType = "job" - case *PersistentVolume: - ss := client.CoreV1().PersistentVolumes() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return ss.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return ss.Watch(ctx, options) - }, - } - - objType = "persistentvolume" - case *PersistentVolumeClaim: - ss := client.CoreV1().PersistentVolumeClaims(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return ss.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return ss.Watch(ctx, options) - }, - } - - objType = "persistentvolumeclaim" - case *Role: - r := client.RbacV1().Roles(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return r.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return r.Watch(ctx, options) - }, - } - - objType = "role" - - case *RoleBinding: - rb := client.RbacV1().RoleBindings(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return rb.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return rb.Watch(ctx, options) - }, - } - - objType = "rolebinding" - - case *ClusterRole: - cr := client.RbacV1().ClusterRoles() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return cr.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return cr.Watch(ctx, options) - }, - } - - objType = "clusterrole" - - case *ClusterRoleBinding: - crb := client.RbacV1().ClusterRoleBindings() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return crb.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return crb.Watch(ctx, options) - }, - } - - objType = "clusterrolebinding" - - case *PodSecurityPolicy: - psp := client.PolicyV1beta1().PodSecurityPolicies() - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return psp.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return psp.Watch(ctx, options) - }, - } - - objType = "podsecuritypolicy" - - case *NetworkPolicy: - np := client.ExtensionsV1beta1().NetworkPolicies(opts.Namespace) - listwatch = &cache.ListWatch{ - ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { - return np.List(ctx, options) - }, - WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { - return np.Watch(ctx, options) - }, - } - - objType = "networkpolicy" - - default: - return nil, "", fmt.Errorf("unsupported resource type for watching %T", resource) - } - - if indexers == nil { - indexers = cache.Indexers{} - } - return cache.NewSharedIndexInformer(listwatch, resource, opts.SyncTimeout, indexers), objType, nil -} diff --git a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go deleted file mode 100644 index 3d8389801e1..00000000000 --- a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore.go +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package k8skeystore - -import ( - "context" - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8s "k8s.io/client-go/kubernetes" - - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -// KubernetesKeystoresRegistry implements a Provider for Keystore. -type KubernetesKeystoresRegistry struct { - logger *logp.Logger - client k8s.Interface -} - -// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace -type KubernetesSecretsKeystore struct { - namespace string - client k8s.Interface - logger *logp.Logger -} - -// Factoryk8s Create the right keystore with the configured options -func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { - keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) - return keystore, err -} - -// NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry -func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) keystore.Provider { - return &KubernetesKeystoresRegistry{ - logger: logger, - client: client, - } -} - -// GetKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. -func (kr *KubernetesKeystoresRegistry) GetKeystore(event bus.Event) keystore.Keystore { - namespace := "" - if val, ok := event["kubernetes"]; ok { - kubernetesMeta := val.(mapstr.M) - ns, err := kubernetesMeta.GetValue("namespace") - if err != nil { - kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) - return nil - } - namespace = ns.(string) - } - if namespace != "" { - k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger) - return k8sKeystore - } - kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) - return nil -} - -// NewKubernetesSecretsKeystore returns an new k8s Keystore -func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { - keystore := KubernetesSecretsKeystore{ - namespace: keystoreNamespace, - client: ks8client, - logger: logger, - } - return &keystore, nil -} - -// Retrieve return a SecureString instance that will contains both the key and the secret. -func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString, error) { - // key = "kubernetes.somenamespace.somesecret.value" - tokens := strings.Split(key, ".") - if len(tokens) > 0 && tokens[0] != "kubernetes" { - return nil, keystore.ErrKeyDoesntExists - } - if len(tokens) != 4 { - k.logger.Debugf( - "not valid secret key: %v. Secrets should be of the following format %v", - key, - "kubernetes.somenamespace.somesecret.value", - ) - return nil, keystore.ErrKeyDoesntExists - } - ns := tokens[1] - secretName := tokens[2] - secretVar := tokens[3] - if ns != k.namespace { - k.logger.Debugf("cannot access Kubernetes secrets from a different namespace (%v) than: %v", ns, k.namespace) - return nil, keystore.ErrKeyDoesntExists - } - secretIntefrace := k.client.CoreV1().Secrets(ns) - ctx := context.TODO() - secret, err := secretIntefrace.Get(ctx, secretName, metav1.GetOptions{}) - if err != nil { - k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) - return nil, keystore.ErrKeyDoesntExists - } - if _, ok := secret.Data[secretVar]; !ok { - k.logger.Errorf("Could not retrieve value %v for secret %v", secretVar, secretName) - return nil, keystore.ErrKeyDoesntExists - } - secretString := secret.Data[secretVar] - return keystore.NewSecureString(secretString), nil -} - -// GetConfig returns config.C representation of the key / secret pair to be merged with other -// loaded configuration. -func (k *KubernetesSecretsKeystore) GetConfig() (*config.C, error) { - return nil, nil -} - -// IsPersisted return if the keystore is physically persisted on disk. -func (k *KubernetesSecretsKeystore) IsPersisted() bool { - return true -} diff --git a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go b/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go deleted file mode 100644 index 13d5f5997cb..00000000000 --- a/libbeat/common/kubernetes/k8skeystore/kubernetes_keystore_test.go +++ /dev/null @@ -1,209 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package k8skeystore - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sfake "k8s.io/client-go/kubernetes/fake" - - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func TestGetKeystore(t *testing.T) { - kRegistry := NewKubernetesKeystoresRegistry(nil, nil) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) - k2 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) - assert.Equal(t, k1, k2) - k3 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace_2"}}) - assert.NotEqual(t, k2, k3) -} - -func TestGetKeystoreAndRetrieve(t *testing.T) { - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - pass := "testing_passpass" - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "apps/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "testing_secret", - Namespace: ns, - }, - Data: map[string][]byte{ - "secret_value": []byte(pass), - }, - } - _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s secret: %v", err) - } - - kRegistry := NewKubernetesKeystoresRegistry(nil, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "kubernetes.test_namespace.testing_secret.secret_value" - secure, err := k1.Retrieve(key) - if err != nil { - t.Fatalf("could not retrive k8s secret: %v", err) - } - secretVal, err := secure.Get() - assert.NoError(t, err) - bytePassword := []byte(pass) - assert.Equal(t, bytePassword, secretVal) -} - -func TestGetKeystoreAndRetrieveWithNonAllowedNamespace(t *testing.T) { - logger := logp.NewLogger("test_k8s_secrets") - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - pass := "testing_passpass" - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "apps/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "testing_secret", - Namespace: ns, - }, - Data: map[string][]byte{ - "secret_value": []byte(pass), - }, - } - _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s secret: %v", err) - } - - kRegistry := NewKubernetesKeystoresRegistry(logger, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "kubernetes.test_namespace_HACK.testing_secret.secret_value" - _, err = k1.Retrieve(key) - assert.Error(t, err) -} - -func TestGetKeystoreAndRetrieveWithWrongKeyFormat(t *testing.T) { - logger := logp.NewLogger("test_k8s_secrets") - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - pass := "testing_passpass" - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "apps/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "testing_secret", - Namespace: ns, - }, - Data: map[string][]byte{ - "secret_value": []byte(pass), - }, - } - _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s secret: %v", err) - } - - kRegistry := NewKubernetesKeystoresRegistry(logger, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "HACK_test_namespace_HACK.testing_secret.secret_value" - _, err = k1.Retrieve(key) - assert.Error(t, err) -} - -func TestGetKeystoreAndRetrieveWithNoSecretsExistent(t *testing.T) { - logger := logp.NewLogger("test_k8s_secrets") - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - - kRegistry := NewKubernetesKeystoresRegistry(logger, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "kubernetes.test_namespace.testing_secret.secret_value" - _, err := k1.Retrieve(key) - assert.Error(t, err) -} - -func TestGetKeystoreAndRetrieveWithWrongSecretName(t *testing.T) { - logger := logp.NewLogger("test_k8s_secrets") - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - pass := "testing_passpass" - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "apps/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "testing_secret", - Namespace: ns, - }, - Data: map[string][]byte{ - "secret_value": []byte(pass), - }, - } - _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s secret: %v", err) - } - - kRegistry := NewKubernetesKeystoresRegistry(logger, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "kubernetes.test_namespace.testing_secret_WRONG.secret_value" - _, err = k1.Retrieve(key) - assert.Error(t, err) -} - -func TestGetKeystoreAndRetrieveWithWrongSecretValue(t *testing.T) { - logger := logp.NewLogger("test_k8s_secrets") - client := k8sfake.NewSimpleClientset() - ns := "test_namespace" - pass := "testing_passpass" - secret := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "apps/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "testing_secret", - Namespace: ns, - }, - Data: map[string][]byte{ - "secret_value": []byte(pass), - }, - } - _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s secret: %v", err) - } - - kRegistry := NewKubernetesKeystoresRegistry(logger, client) - k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) - key := "kubernetes.test_namespace.testing_secret.secret_value_WRONG" - _, err = k1.Retrieve(key) - assert.Error(t, err) -} diff --git a/libbeat/common/kubernetes/metadata/config.go b/libbeat/common/kubernetes/metadata/config.go deleted file mode 100644 index 126610d6ac7..00000000000 --- a/libbeat/common/kubernetes/metadata/config.go +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import "github.com/elastic/elastic-agent-libs/config" - -// Config declares supported configuration for metadata generation -type Config struct { - KubeConfig string `config:"kube_config"` - IncludeLabels []string `config:"include_labels"` - ExcludeLabels []string `config:"exclude_labels"` - IncludeAnnotations []string `config:"include_annotations"` - - LabelsDedot bool `config:"labels.dedot"` - AnnotationsDedot bool `config:"annotations.dedot"` -} - -// AddResourceMetadataConfig allows adding config for enriching additional resources -type AddResourceMetadataConfig struct { - Node *config.C `config:"node"` - Namespace *config.C `config:"namespace"` - Deployment bool `config:"deployment"` - CronJob bool `config:"cronjob"` -} - -// InitDefaults initializes the defaults for the config. -func (c *Config) InitDefaults() { - c.LabelsDedot = true - c.AnnotationsDedot = true -} - -// Unmarshal unpacks a Config into the metagen Config -func (c *Config) Unmarshal(cfg *config.C) error { - return cfg.Unpack(c) -} - -func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig { - metaConfig := Config{} - metaConfig.InitDefaults() - metaCfg, _ := config.NewConfigFrom(&metaConfig) - return &AddResourceMetadataConfig{ - Node: metaCfg, - Namespace: metaCfg, - Deployment: true, - CronJob: true, - } -} diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go deleted file mode 100644 index 08d3472c17b..00000000000 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "context" - "fmt" - "strings" - - "gopkg.in/yaml.v2" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8sclient "k8s.io/client-go/kubernetes" - - "k8s.io/client-go/tools/clientcmd" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/safemapstr" -) - -// MetaGen allows creation of metadata from either Kubernetes resources or their Resource names. -type MetaGen interface { - // Generate generates metadata for a given resource. - // Metadata map is formed in the following format: - // { - // "kubernetes": GenerateK8s(), - // "some.ecs.field": "asdf, // populated by GenerateECS() - // } - // This method is called in top level and returns the complete map of metadata. - Generate(kubernetes.Resource, ...FieldOptions) mapstr.M - // GenerateFromName generates metadata for a given resource based on it's name - GenerateFromName(string, ...FieldOptions) mapstr.M - // GenerateK8s generates kubernetes metadata for a given resource - GenerateK8s(kubernetes.Resource, ...FieldOptions) mapstr.M - // GenerateECS generates ECS metadata for a given resource - GenerateECS(kubernetes.Resource) mapstr.M -} - -// FieldOptions allows additional enrichment to be done on top of existing metadata -type FieldOptions func(mapstr.M) - -type ClusterInfo struct { - Url string - Name string -} - -type ClusterConfiguration struct { - ControlPlaneEndpoint string `yaml:"controlPlaneEndpoint"` - ClusterName string `yaml:"clusterName"` -} - -// WithFields FieldOption allows adding specific fields into the generated metadata -func WithFields(key string, value interface{}) FieldOptions { - return func(meta mapstr.M) { - safemapstr.Put(meta, key, value) - } -} - -// WithMetadata FieldOption allows adding labels and annotations under sub-resource(kind) -// example if kind=namespace namespace.labels key will be added -func WithMetadata(kind string) FieldOptions { - return func(meta mapstr.M) { - if meta["labels"] != nil { - safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) - } - if meta["annotations"] != nil { - safemapstr.Put(meta, strings.ToLower(kind)+".annotations", meta["annotations"]) - } - } -} - -// GetPodMetaGen is a wrapper function that creates a metaGen for pod resource and has embeeded -// nodeMetaGen and namespaceMetaGen -func GetPodMetaGen( - cfg *config.C, - podWatcher kubernetes.Watcher, - nodeWatcher kubernetes.Watcher, - namespaceWatcher kubernetes.Watcher, - metaConf *AddResourceMetadataConfig) MetaGen { - - var nodeMetaGen, namespaceMetaGen MetaGen - if nodeWatcher != nil && metaConf.Node.Enabled() { - nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store(), nodeWatcher.Client()) - } - if namespaceWatcher != nil && metaConf.Namespace.Enabled() { - namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client()) - } - metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf) - return metaGen -} - -// GetKubernetesClusterIdentifier returns ClusterInfo for k8s if available -func GetKubernetesClusterIdentifier(cfg *config.C, client k8sclient.Interface) (ClusterInfo, error) { - // try with kube config file - var c Config - c.Unmarshal(cfg) - clusterInfo, err := getClusterInfoFromKubeConfigFile(c.KubeConfig) - if err == nil { - return clusterInfo, nil - } - // try with kubeadm-config configmap - clusterInfo, err = getClusterInfoFromKubeadmConfigMap(client) - if err == nil { - return clusterInfo, nil - } - return ClusterInfo{}, fmt.Errorf("unable to retrieve cluster identifiers") -} - -func getClusterInfoFromKubeadmConfigMap(client k8sclient.Interface) (ClusterInfo, error) { - clusterInfo := ClusterInfo{} - if client == nil { - return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config") - } - cm, err := client.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kubeadm-config", metav1.GetOptions{}) - if err != nil { - return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config: %+v", err) - } - p, ok := cm.Data["ClusterConfiguration"] - if !ok { - return clusterInfo, fmt.Errorf("unable to get cluster identifiers from ClusterConfiguration") - } - - cc := &ClusterConfiguration{} - err = yaml.Unmarshal([]byte(p), cc) - if err != nil { - return ClusterInfo{}, err - } - if cc.ClusterName != "" { - clusterInfo.Name = cc.ClusterName - } - if cc.ControlPlaneEndpoint != "" { - clusterInfo.Url = cc.ControlPlaneEndpoint - } - - return clusterInfo, nil -} - -func getClusterInfoFromKubeConfigFile(kubeconfig string) (ClusterInfo, error) { - if kubeconfig == "" { - kubeconfig = kubernetes.GetKubeConfigEnvironmentVariable() - } - - if kubeconfig == "" { - return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config from env") - } - - cfg, err := kubernetes.BuildConfig(kubeconfig) - if err != nil { - return ClusterInfo{}, fmt.Errorf("unable to build kube config due to error: %+v", err) - } - - kube_cfg, err := clientcmd.LoadFromFile(kubeconfig) - if err != nil { - return ClusterInfo{}, fmt.Errorf("unable to load kube_config due to error: %+v", err) - } - - for key, element := range kube_cfg.Clusters { - if element.Server == cfg.Host { - return ClusterInfo{element.Server, key}, nil - } - } - return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config") -} diff --git a/libbeat/common/kubernetes/metadata/namespace.go b/libbeat/common/kubernetes/metadata/namespace.go deleted file mode 100644 index efd181eb11d..00000000000 --- a/libbeat/common/kubernetes/metadata/namespace.go +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - k8s "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -const resource = "namespace" - -type namespace struct { - store cache.Store - resource *Resource -} - -// NewNamespaceMetadataGenerator creates a metagen for namespace resources -func NewNamespaceMetadataGenerator(cfg *config.C, namespaces cache.Store, client k8s.Interface) MetaGen { - return &namespace{ - resource: NewResourceMetadataGenerator(cfg, client), - store: namespaces, - } -} - -// Generate generates pod metadata from a resource object -// Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } -// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by -// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method -func (n *namespace) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := n.GenerateECS(obj) - meta := mapstr.M{ - "kubernetes": n.GenerateK8s(obj, opts...), - } - meta.DeepUpdate(ecsFields) - return meta -} - -// GenerateECS generates namespace ECS metadata from a resource object -func (n *namespace) GenerateECS(obj kubernetes.Resource) mapstr.M { - return n.resource.GenerateECS(obj) -} - -// GenerateK8s generates namespace metadata from a resource object -func (n *namespace) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - _, ok := obj.(*kubernetes.Namespace) - if !ok { - return nil - } - - meta := n.resource.GenerateK8s(resource, obj, opts...) - meta = flattenMetadata(meta) - - // TODO: Add extra fields in here if need be - return meta -} - -// GenerateFromName generates pod metadata from a namespace name -func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { - if n.store == nil { - return nil - } - - if obj, ok, _ := n.store.GetByKey(name); ok { - no, ok := obj.(*kubernetes.Namespace) - if !ok { - return nil - } - - return n.GenerateK8s(no, opts...) - } - - return nil -} - -func flattenMetadata(in mapstr.M) mapstr.M { - out := mapstr.M{} - rawFields, err := in.GetValue(resource) - if err != nil { - return nil - } - - fields, ok := rawFields.(mapstr.M) - if !ok { - return nil - } - for k, v := range fields { - if k == "name" { - out[resource] = v - } else { - out[resource+"_"+k] = v - } - } - - populateFromKeys := []string{"labels", "annotations"} - for _, key := range populateFromKeys { - rawValues, err := in.GetValue(key) - if err != nil { - continue - } - values, ok := rawValues.(mapstr.M) - if ok { - out[resource+"_"+key] = values - } - } - - return out -} diff --git a/libbeat/common/kubernetes/metadata/namespace_test.go b/libbeat/common/kubernetes/metadata/namespace_test.go deleted file mode 100644 index c10d59bdcb2..00000000000 --- a/libbeat/common/kubernetes/metadata/namespace_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - k8sfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func TestNamespace_Generate(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - name := "obj" - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Labels: map[string]string{ - "foo": "bar", - "key": "value", - }, - Annotations: map[string]string{ - "spam": "baz", - "key": "value", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - }, - output: mapstr.M{"kubernetes": mapstr.M{ - "namespace": name, - "namespace_uid": uid, - "namespace_labels": mapstr.M{ - "foo": "bar", - }, - "namespace_annotations": mapstr.M{ - "spam": "baz", - }, - }}, - }, - } - - cfg, err := config.NewConfigFrom(Config{ - IncludeLabels: []string{"foo"}, - IncludeAnnotations: []string{"spam"}, - }) - if err != nil { - t.Fatalf("Could not merge configs") - } - - metagen := NewNamespaceMetadataGenerator(cfg, nil, client) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} - -func TestNamespace_GenerateFromName(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - name := "obj" - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "spam": "baz", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "namespace": name, - "namespace_uid": uid, - "namespace_labels": mapstr.M{ - "foo": "bar", - }, - "namespace_annotations": mapstr.M{ - "spam": "baz", - }, - }, - }, - } - - for _, test := range tests { - cfg, err := config.NewConfigFrom(Config{ - IncludeAnnotations: []string{"spam"}, - }) - if err != nil { - t.Fatalf("Could not merge configs") - } - - namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) - namespaces.Add(test.input) - metagen := NewNamespaceMetadataGenerator(cfg, namespaces, client) - - accessor, err := meta.Accessor(test.input) - require.NoError(t, err) - - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) - }) - } -} diff --git a/libbeat/common/kubernetes/metadata/node.go b/libbeat/common/kubernetes/metadata/node.go deleted file mode 100644 index 839cf4252ba..00000000000 --- a/libbeat/common/kubernetes/metadata/node.go +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - v1 "k8s.io/api/core/v1" - k8s "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -type node struct { - store cache.Store - resource *Resource -} - -// NewNodeMetadataGenerator creates a metagen for service resources -func NewNodeMetadataGenerator(cfg *config.C, nodes cache.Store, client k8s.Interface) MetaGen { - return &node{ - resource: NewResourceMetadataGenerator(cfg, client), - store: nodes, - } -} - -// Generate generates node metadata from a resource object -// Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } -// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by -// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method -func (n *node) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := n.GenerateECS(obj) - meta := mapstr.M{ - "kubernetes": n.GenerateK8s(obj, opts...), - } - meta.DeepUpdate(ecsFields) - return meta -} - -// GenerateECS generates node ECS metadata from a resource object -func (n *node) GenerateECS(obj kubernetes.Resource) mapstr.M { - return n.resource.GenerateECS(obj) -} - -// GenerateK8s generates node metadata from a resource object -func (n *node) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - node, ok := obj.(*kubernetes.Node) - if !ok { - return nil - } - - meta := n.resource.GenerateK8s("node", obj, opts...) - // TODO: Add extra fields in here if need be - hostname := getHostName(node) - if hostname != "" { - meta.Put("node.hostname", hostname) - } - return meta -} - -// GenerateFromName generates pod metadata from a service name -func (n *node) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { - if n.store == nil { - return nil - } - - if obj, ok, _ := n.store.GetByKey(name); ok { - no, ok := obj.(*kubernetes.Node) - if !ok { - return nil - } - - return n.GenerateK8s(no, opts...) - } - - return nil -} - -// getHostName returns the HostName address of the node -func getHostName(node *v1.Node) string { - for _, adr := range node.Status.Addresses { - if adr.Type == v1.NodeHostName { - return adr.Address - } - } - return "" -} diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go deleted file mode 100644 index ab98cbc7234..00000000000 --- a/libbeat/common/kubernetes/metadata/node_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - k8sfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func TestNode_Generate(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - name := "obj" - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, - }, - }, - output: mapstr.M{"kubernetes": mapstr.M{ - "node": mapstr.M{ - "name": "obj", - "uid": uid, - "hostname": "node1", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "key2": "value2", - }, - }}, - }, - } - - cfg, _ := config.NewConfigFrom(Config{ - IncludeAnnotations: []string{"key2"}, - }) - metagen := NewNodeMetadataGenerator(cfg, nil, client) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} - -func TestNode_GenerateFromName(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - name := "obj" - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "key": "value", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, - }, - }, - output: mapstr.M{ - "node": mapstr.M{ - "name": "obj", - "uid": uid, - "hostname": "node1", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "key": "value", - }, - }, - }, - } - - for _, test := range tests { - cfg, _ := config.NewConfigFrom(Config{ - IncludeAnnotations: []string{"key"}, - }) - nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) - nodes.Add(test.input) - metagen := NewNodeMetadataGenerator(cfg, nodes, client) - - accessor, err := meta.Accessor(test.input) - require.NoError(t, err) - - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) - }) - } -} diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go deleted file mode 100644 index 096d2757ba4..00000000000 --- a/libbeat/common/kubernetes/metadata/pod.go +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "context" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8s "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -type pod struct { - store cache.Store - client k8s.Interface - node MetaGen - namespace MetaGen - resource *Resource - addResourceMetadata *AddResourceMetadataConfig -} - -// NewPodMetadataGenerator creates a metagen for pod resources -func NewPodMetadataGenerator( - cfg *config.C, - pods cache.Store, - client k8s.Interface, - node MetaGen, - namespace MetaGen, - addResourceMetadata *AddResourceMetadataConfig) MetaGen { - - return &pod{ - resource: NewResourceMetadataGenerator(cfg, client), - store: pods, - node: node, - namespace: namespace, - client: client, - addResourceMetadata: addResourceMetadata, - } -} - -// Generate generates pod metadata from a resource object -// Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } -// All Kubernetes fields that need to be stored under kubernetes. prefix are populated by -// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method -func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := p.GenerateECS(obj) - meta := mapstr.M{ - "kubernetes": p.GenerateK8s(obj, opts...), - } - meta.DeepUpdate(ecsFields) - return meta -} - -// GenerateECS generates pod ECS metadata from a resource object -func (p *pod) GenerateECS(obj kubernetes.Resource) mapstr.M { - return p.resource.GenerateECS(obj) -} - -// GenerateK8s generates pod metadata from a resource object -func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - po, ok := obj.(*kubernetes.Pod) - if !ok { - return nil - } - - out := p.resource.GenerateK8s("pod", obj, opts...) - - // check if Pod is handled by a ReplicaSet which is controlled by a Deployment - if p.addResourceMetadata.Deployment { - rsName, _ := out.GetValue("replicaset.name") - if rsName, ok := rsName.(string); ok { - dep := p.getRSDeployment(rsName, po.GetNamespace()) - if dep != "" { - out.Put("deployment.name", dep) - } - } - } - - // check if Pod is handled by a Job which is controlled by a CronJob - if p.addResourceMetadata.CronJob { - jobName, _ := out.GetValue("job.name") - if jobName, ok := jobName.(string); ok { - dep := p.getCronjobOfJob(jobName, po.GetNamespace()) - if dep != "" { - out.Put("cronjob.name", dep) - } - } - } - - if p.node != nil { - meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node")) - if meta != nil { - out.Put("node", meta["node"]) - } else { - out.Put("node.name", po.Spec.NodeName) - } - } else { - out.Put("node.name", po.Spec.NodeName) - } - - if p.namespace != nil { - meta := p.namespace.GenerateFromName(po.GetNamespace()) - if meta != nil { - out.DeepUpdate(meta) - } - } - - if po.Status.PodIP != "" { - out.Put("pod.ip", po.Status.PodIP) - } - - return out -} - -// GenerateFromName generates pod metadata from a pod name -func (p *pod) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { - if p.store == nil { - return nil - } - - if obj, ok, _ := p.store.GetByKey(name); ok { - po, ok := obj.(*kubernetes.Pod) - if !ok { - return nil - } - - return p.GenerateK8s(po, opts...) - } - - return nil -} - -// getRSDeployment return the name of the Deployment object that -// owns the ReplicaSet with the given name under the given Namespace -func (p *pod) getRSDeployment(rsName string, ns string) string { - if p.client == nil { - return "" - } - rs, err := p.client.AppsV1().ReplicaSets(ns).Get(context.TODO(), rsName, metav1.GetOptions{}) - if err != nil { - return "" - } - for _, ref := range rs.GetOwnerReferences() { - if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - case "Deployment": - return ref.Name - } - } - } - return "" -} - -// getCronjobOfJob return the name of the Cronjob object that -// owns the Job with the given name under the given Namespace -func (p *pod) getCronjobOfJob(jobName string, ns string) string { - if p.client == nil { - return "" - } - cronjob, err := p.client.BatchV1().Jobs(ns).Get(context.TODO(), jobName, metav1.GetOptions{}) - if err != nil { - return "" - } - for _, ref := range cronjob.GetOwnerReferences() { - if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - case "CronJob": - return ref.Name - } - } - } - return "" -} diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go deleted file mode 100644 index 65aae067730..00000000000 --- a/libbeat/common/kubernetes/metadata/pod_test.go +++ /dev/null @@ -1,845 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - k8sfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -var addResourceMetadata = GetDefaultResourceMetadataConfig() - -func TestPod_Generate(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - rs := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx-rs", - Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "nginx-deployment", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - Spec: appsv1.ReplicaSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "demo", - }, - }, - Template: v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "demo", - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "nginx", - Image: "nginx:1.12", - Ports: []v1.ContainerPort{ - { - Name: "http", - Protocol: v1.ProtocolTCP, - ContainerPort: 80, - }, - }, - }, - }, - }, - }, - }, - } - - _, err := client.AppsV1().ReplicaSets(namespace).Create(context.Background(), rs, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("failed to create k8s deployment: %v", err) - } - - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - "namespace": "default", - "node": mapstr.M{ - "name": "testnode", - }, - }, - }, - }, - { - name: "test object with owner reference to Deployment", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "owner", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }, - }, - }, - { - name: "test object with owner reference to DaemonSet", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "DaemonSet", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "daemonset": mapstr.M{ - "name": "owner", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }, - }, - }, - { - name: "test object with owner reference to Job", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "batch/v1", - Kind: "Job", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "job": mapstr.M{ - "name": "owner", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }, - }, - }, - { - name: "test object with owner reference to replicaset", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "ReplicaSet", - Name: "nginx-rs", - UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "nginx-deployment", - }, - "replicaset": mapstr.M{ - "name": "nginx-rs", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }, - }, - }, - { - name: "test object with owner reference to replicaset honors annotations.dedot: false", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "k8s.app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "ReplicaSet", - Name: "nginx-rs", - UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "nginx-deployment", - }, - "replicaset": mapstr.M{ - "name": "nginx-rs", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "k8s": mapstr.M{"app": "production"}, - }, - }, - }, - }, - } - - config, err := conf.NewConfigFrom(map[string]interface{}{ - "include_annotations": []string{"app", "k8s.app"}, - "annotations.dedot": false, - }) - assert.NoError(t, err) - - metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, addResourceMetadata) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} - -func TestPod_GenerateFromName(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "k8s.app": "production", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "k8s_app": "production", - }, - }, - }, - { - name: "test object with owner reference", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - output: mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "owner", - }, - "node": mapstr.M{ - "name": "testnode", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }, - }, - } - - for _, test := range tests { - config, err := conf.NewConfigFrom(map[string]interface{}{ - "include_annotations": []string{"app", "k8s.app"}, - }) - assert.NoError(t, err) - pods := cache.NewStore(cache.MetaNamespaceKeyFunc) - pods.Add(test.input) - metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, addResourceMetadata) - - accessor, err := meta.Accessor(test.input) - require.NoError(t, err) - - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) - }) - } -} - -func TestPod_GenerateWithNodeNamespace(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - tests := []struct { - input kubernetes.Resource - node kubernetes.Resource - namespace kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{ - "app": "production", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - UID: types.UID(uid), - Labels: map[string]string{ - "nodekey": "nodevalue", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, - }, - }, - namespace: &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - UID: types.UID(uid), - Labels: map[string]string{ - "nskey": "nsvalue", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - }, - output: mapstr.M{"kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "namespace_uid": uid, - "namespace_labels": mapstr.M{ - "nskey": "nsvalue", - }, - "node": mapstr.M{ - "name": "testnode", - "uid": uid, - "labels": mapstr.M{ - "nodekey": "nodevalue", - }, - "hostname": "node1", - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "annotations": mapstr.M{ - "app": "production", - }, - }}, - }, - } - - for _, test := range tests { - config, err := conf.NewConfigFrom(map[string]interface{}{ - "include_annotations": []string{"app"}, - }) - assert.NoError(t, err) - pods := cache.NewStore(cache.MetaNamespaceKeyFunc) - pods.Add(test.input) - - nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) - nodes.Add(test.node) - nodeMeta := NewNodeMetadataGenerator(config, nodes, client) - - namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) - namespaces.Add(test.namespace) - nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client) - - metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, addResourceMetadata) - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} - -func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - - tests := []struct { - input kubernetes.Resource - node kubernetes.Resource - namespace kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "app.kubernetes.io/component": "exporter", - }, - Annotations: map[string]string{ - "app": "production", - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "ReplicaSet", - Name: "nginx-rs", - UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - - Spec: v1.PodSpec{ - NodeName: "testnode", - }, - Status: v1.PodStatus{PodIP: "127.0.0.5"}, - }, - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - UID: types.UID(uid), - Labels: map[string]string{ - "nodekey": "nodevalue", - "nodekey2": "nodevalue2", - }, - Annotations: map[string]string{ - "node.annotation": "node.value", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - Status: v1.NodeStatus{ - Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, - }, - }, - namespace: &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - UID: types.UID(uid), - Labels: map[string]string{ - "app.kubernetes.io/name": "kube-state-metrics", - "nskey2": "nsvalue2", - }, - Annotations: map[string]string{ - "ns.annotation": "ns.value", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - }, - output: mapstr.M{"kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - "ip": "127.0.0.5", - }, - "namespace": "default", - "namespace_uid": uid, - "namespace_labels": mapstr.M{ - "app_kubernetes_io/name": "kube-state-metrics", - }, - "namespace_annotations": mapstr.M{ - "ns_annotation": "ns.value", - }, - "node": mapstr.M{ - "name": "testnode", - "uid": uid, - "labels": mapstr.M{ - "nodekey2": "nodevalue2", - }, - "hostname": "node1", - "annotations": mapstr.M{ - "node_annotation": "node.value", - }, - }, - "labels": mapstr.M{ - "app_kubernetes_io/component": "exporter", - }, - "annotations": mapstr.M{ - "app": "production", - }, - "replicaset": mapstr.M{ - "name": "nginx-rs", - }, - }}, - }, - } - - for _, test := range tests { - config, err := conf.NewConfigFrom(map[string]interface{}{ - "include_annotations": []string{"app"}, - }) - - assert.NoError(t, err) - - namespaceConfig, _ := conf.NewConfigFrom(map[string]interface{}{ - "include_labels": []string{"app.kubernetes.io/name"}, - "include_annotations": []string{"ns.annotation"}, - }) - nodeConfig, _ := conf.NewConfigFrom(map[string]interface{}{ - "include_labels": []string{"nodekey2"}, - "include_annotations": []string{"node.annotation"}, - }) - metaConfig := AddResourceMetadataConfig{ - Namespace: namespaceConfig, - Node: nodeConfig, - Deployment: false, - } - - pods := cache.NewStore(cache.MetaNamespaceKeyFunc) - pods.Add(test.input) - - nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) - nodes.Add(test.node) - nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client) - - namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) - namespaces.Add(test.namespace) - nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client) - - metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, &metaConfig) - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go deleted file mode 100644 index 92789569ab2..00000000000 --- a/libbeat/common/kubernetes/metadata/resource.go +++ /dev/null @@ -1,183 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "strings" - - "k8s.io/apimachinery/pkg/api/meta" - k8s "k8s.io/client-go/kubernetes" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/safemapstr" -) - -// Resource generates metadata for any kubernetes resource -type Resource struct { - config *Config - clusterInfo ClusterInfo -} - -// NewResourceMetadataGenerator creates a metadata generator for a generic resource -func NewResourceMetadataGenerator(cfg *config.C, client k8s.Interface) *Resource { - var config Config - config.Unmarshal(cfg) - - r := &Resource{ - config: &config, - } - clusterInfo, err := GetKubernetesClusterIdentifier(cfg, client) - if err == nil { - r.clusterInfo = clusterInfo - } - return r -} - -// Generate generates metadata from a resource object -// Generate method returns metadata in the following form: -// { -// "kubernetes": {}, -// "ecs.a.field": 42, -// } -// This method should be called in top level and not as part of other metadata generators. -// For retrieving metadata without kubernetes. prefix one should call GenerateK8s instead. -func (r *Resource) Generate(kind string, obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := r.GenerateECS(obj) - meta := mapstr.M{ - "kubernetes": r.GenerateK8s(kind, obj, opts...), - } - meta.DeepUpdate(ecsFields) - return meta -} - -// GenerateECS generates ECS metadata from a resource object -func (r *Resource) GenerateECS(obj kubernetes.Resource) mapstr.M { - ecsMeta := mapstr.M{} - if r.clusterInfo.Url != "" { - ecsMeta.Put("orchestrator.cluster.url", r.clusterInfo.Url) - } - if r.clusterInfo.Name != "" { - ecsMeta.Put("orchestrator.cluster.name", r.clusterInfo.Name) - } - return ecsMeta -} - -// GenerateK8s takes a kind and an object and creates metadata for the same -func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ...FieldOptions) mapstr.M { - accessor, err := meta.Accessor(obj) - if err != nil { - return nil - } - - var labelMap mapstr.M - if len(r.config.IncludeLabels) == 0 { - labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot) - } else { - labelMap = generateMapSubset(accessor.GetLabels(), r.config.IncludeLabels, r.config.LabelsDedot) - } - - // Exclude any labels that are present in the exclude_labels config - for _, label := range r.config.ExcludeLabels { - labelMap.Delete(label) - } - - annotationsMap := generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.AnnotationsDedot) - - meta := mapstr.M{ - strings.ToLower(kind): mapstr.M{ - "name": accessor.GetName(), - "uid": string(accessor.GetUID()), - }, - } - - if accessor.GetNamespace() != "" { - safemapstr.Put(meta, "namespace", accessor.GetNamespace()) - } - - // Add controller metadata if present - for _, ref := range accessor.GetOwnerReferences() { - if ref.Controller != nil && *ref.Controller { - switch ref.Kind { - // TODO grow this list as we keep adding more `state_*` metricsets - case "Deployment", - "ReplicaSet", - "StatefulSet", - "DaemonSet", - "Job", - "CronJob": - safemapstr.Put(meta, strings.ToLower(ref.Kind)+".name", ref.Name) - } - } - } - - if len(labelMap) != 0 { - safemapstr.Put(meta, "labels", labelMap) - } - - if len(annotationsMap) != 0 { - safemapstr.Put(meta, "annotations", annotationsMap) - } - - for _, option := range options { - option(meta) - } - - return meta -} - -func generateMapSubset(input map[string]string, keys []string, dedot bool) mapstr.M { - output := mapstr.M{} - if input == nil { - return output - } - - for _, key := range keys { - value, ok := input[key] - if ok { - if dedot { - dedotKey := common.DeDot(key) - output.Put(dedotKey, value) - } else { - safemapstr.Put(output, key, value) - } - } - } - - return output -} - -func GenerateMap(input map[string]string, dedot bool) mapstr.M { - output := mapstr.M{} - if input == nil { - return output - } - - for k, v := range input { - if dedot { - label := common.DeDot(k) - output.Put(label, v) - } else { - safemapstr.Put(output, k, v) - } - } - - return output -} diff --git a/libbeat/common/kubernetes/metadata/resource_test.go b/libbeat/common/kubernetes/metadata/resource_test.go deleted file mode 100644 index ee33fb35ce7..00000000000 --- a/libbeat/common/kubernetes/metadata/resource_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "testing" - - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/go-ucfg" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" -) - -func TestResource_Generate(t *testing.T) { - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "namespace": "default", - }, - }, - }, - { - name: "test object with owner reference", - input: &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Pod", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "pod": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "owner", - }, - }, - }, - }, - } - - var cfg Config - ucfg.New().Unpack(&cfg) - metagen := &Resource{ - config: &cfg, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate("pod", test.input)) - }) - } -} diff --git a/libbeat/common/kubernetes/metadata/service.go b/libbeat/common/kubernetes/metadata/service.go deleted file mode 100644 index e9243ade69f..00000000000 --- a/libbeat/common/kubernetes/metadata/service.go +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - k8s "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/safemapstr" -) - -type service struct { - store cache.Store - namespace MetaGen - resource *Resource -} - -// NewServiceMetadataGenerator creates a metagen for service resources -func NewServiceMetadataGenerator(cfg *config.C, services cache.Store, namespace MetaGen, client k8s.Interface) MetaGen { - return &service{ - resource: NewResourceMetadataGenerator(cfg, client), - store: services, - namespace: namespace, - } -} - -// Generate generates service metadata from a resource object -// Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } -// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by -// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method -func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - ecsFields := s.GenerateECS(obj) - meta := mapstr.M{ - "kubernetes": s.GenerateK8s(obj, opts...), - } - meta.DeepUpdate(ecsFields) - return meta -} - -// GenerateECS generates service ECS metadata from a resource object -func (s *service) GenerateECS(obj kubernetes.Resource) mapstr.M { - return s.resource.GenerateECS(obj) -} - -// GenerateK8s generates service metadata from a resource object -func (s *service) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { - svc, ok := obj.(*kubernetes.Service) - if !ok { - return nil - } - - out := s.resource.GenerateK8s("service", obj, opts...) - - if s.namespace != nil { - meta := s.namespace.GenerateFromName(svc.GetNamespace()) - if meta != nil { - // Use this in 8.0 - //out.Put("namespace", meta["namespace"]) - out.DeepUpdate(meta) - } - } - - selectors := svc.Spec.Selector - if len(selectors) == 0 { - return out - } - svcMap := GenerateMap(selectors, s.resource.config.LabelsDedot) - if len(svcMap) != 0 { - safemapstr.Put(out, "selectors", svcMap) - } - - return out -} - -// GenerateFromName generates pod metadata from a service name -func (s *service) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { - if s.store == nil { - return nil - } - - if obj, ok, _ := s.store.GetByKey(name); ok { - svc, ok := obj.(*kubernetes.Service) - if !ok { - return nil - } - - return s.GenerateK8s(svc, opts...) - } - - return nil -} diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go deleted file mode 100644 index 185e4d2d4d3..00000000000 --- a/libbeat/common/kubernetes/metadata/service_test.go +++ /dev/null @@ -1,330 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package metadata - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - k8sfake "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" -) - -func TestService_Generate(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - Spec: v1.ServiceSpec{ - Selector: map[string]string{ - "app": "istiod", - "istio": "pilot", - }, - }, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "service": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "selectors": mapstr.M{ - "app": "istiod", - "istio": "pilot", - }, - "namespace": "default", - }, - }, - }, - { - name: "test object with owner reference", - input: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - Spec: v1.ServiceSpec{ - Selector: map[string]string{ - "app": "istiod", - "istio": "pilot", - }, - }, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "service": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "selectors": mapstr.M{ - "app": "istiod", - "istio": "pilot", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "owner", - }, - }, - }, - }, - } - - cfg := config.NewConfig() - metagen := NewServiceMetadataGenerator(cfg, nil, nil, client) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} - -func TestService_GenerateFromName(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - boolean := true - tests := []struct { - input kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "service": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "namespace": "default", - }, - }, - { - name: "test object with owner reference", - input: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps", - Kind: "Deployment", - Name: "owner", - UID: "005f3b90-4b9d-12f8-acf0-31020a840144", - Controller: &boolean, - }, - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "service": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "namespace": "default", - "deployment": mapstr.M{ - "name": "owner", - }, - }, - }, - } - - for _, test := range tests { - cfg := config.NewConfig() - services := cache.NewStore(cache.MetaNamespaceKeyFunc) - services.Add(test.input) - metagen := NewServiceMetadataGenerator(cfg, services, nil, client) - - accessor, err := meta.Accessor(test.input) - require.NoError(t, err) - - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) - }) - } -} - -func TestService_GenerateWithNamespace(t *testing.T) { - client := k8sfake.NewSimpleClientset() - uid := "005f3b90-4b9d-12f8-acf0-31020a840133" - namespace := "default" - name := "obj" - tests := []struct { - input kubernetes.Resource - namespace kubernetes.Resource - output mapstr.M - name string - }{ - { - name: "test simple object", - input: &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - UID: types.UID(uid), - Namespace: namespace, - Labels: map[string]string{ - "foo": "bar", - }, - Annotations: map[string]string{}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - }, - namespace: &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - UID: types.UID(uid), - Labels: map[string]string{ - "nskey": "nsvalue", - }, - Annotations: map[string]string{ - "ns.annotation": "value", - }, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Namespace", - APIVersion: "v1", - }, - }, - output: mapstr.M{ - "kubernetes": mapstr.M{ - "service": mapstr.M{ - "name": "obj", - "uid": uid, - }, - "labels": mapstr.M{ - "foo": "bar", - }, - "namespace": "default", - "namespace_uid": uid, - "namespace_labels": mapstr.M{ - "nskey": "nsvalue", - }, - "namespace_annotations": mapstr.M{ - "ns_annotation": "value", - }, - }, - }, - }, - } - - for _, test := range tests { - nsConfig, _ := config.NewConfigFrom(map[string]interface{}{ - "include_annotations": []string{"ns.annotation"}, - }) - services := cache.NewStore(cache.MetaNamespaceKeyFunc) - services.Add(test.input) - - namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) - namespaces.Add(test.namespace) - nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) - - metagen := NewServiceMetadataGenerator(nsConfig, services, nsMeta, client) - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.output, metagen.Generate(test.input)) - }) - } -} diff --git a/libbeat/common/kubernetes/options.go b/libbeat/common/kubernetes/options.go deleted file mode 100644 index 63850f3f1cc..00000000000 --- a/libbeat/common/kubernetes/options.go +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -type KubeClientOptions struct { - QPS float32 `config:"qps"` - Burst int `config:"burst"` -} diff --git a/libbeat/common/kubernetes/types.go b/libbeat/common/kubernetes/types.go deleted file mode 100644 index fffcb234b02..00000000000 --- a/libbeat/common/kubernetes/types.go +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "strings" - "time" - - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - v1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" - policyv1beta1 "k8s.io/api/policy/v1beta1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -// Resource data -type Resource = runtime.Object - -// ObjectMeta data -type ObjectMeta = metav1.ObjectMeta - -// Pod data -type Pod = v1.Pod - -// PodSpec data -type PodSpec = v1.PodSpec - -// PodStatus data -type PodStatus = v1.PodStatus - -// Node data -type Node = v1.Node - -// Namespace data -type Namespace = v1.Namespace - -// Container data -type Container = v1.Container - -// ContainerPort data -type ContainerPort = v1.ContainerPort - -// Event data -type Event = v1.Event - -// PodContainerStatus data -type PodContainerStatus = v1.ContainerStatus - -// Deployment data -type Deployment = appsv1.Deployment - -// ReplicaSet data -type ReplicaSet = appsv1.ReplicaSet - -// StatefulSet data -type StatefulSet = appsv1.StatefulSet - -// DaemonSet data -type DaemonSet = appsv1.DaemonSet - -// Service data -type Service = v1.Service - -// ServiceAccount data -type ServiceAccount = v1.ServiceAccount - -// Job data -type Job = batchv1.Job - -// CronJob data -type CronJob = batchv1.CronJob - -// PersistentVolume data -type PersistentVolume = v1.PersistentVolume - -// PersistentVolumeClaim data -type PersistentVolumeClaim = v1.PersistentVolumeClaim - -// Role data -type Role = rbacv1.Role - -// RoleBinding data -type RoleBinding = rbacv1.RoleBinding - -// ClusterRole data -type ClusterRole = rbacv1.ClusterRole - -// ClusterRoleBinding data -type ClusterRoleBinding = rbacv1.ClusterRoleBinding - -// PodSecurityPolicy data -type PodSecurityPolicy = policyv1beta1.PodSecurityPolicy - -// NetworkPolicy data -type NetworkPolicy = networkingv1.NetworkPolicy - -const ( - // PodPending phase - PodPending = v1.PodPending - // PodRunning phase - PodRunning = v1.PodRunning - // PodSucceeded phase - PodSucceeded = v1.PodSucceeded - // PodFailed phase - PodFailed = v1.PodFailed - // PodUnknown phase - PodUnknown = v1.PodUnknown -) - -// Time extracts time from k8s.Time type -func Time(t *metav1.Time) time.Time { - return t.Time -} - -// MicroTime extracts time from k8s.MicroTime type -func MicroTime(t *metav1.MicroTime) time.Time { - return t.Time -} - -// ContainerID parses the container ID to get the actual ID string -func ContainerID(s PodContainerStatus) string { - cID, _ := ContainerIDWithRuntime(s) - return cID -} - -// ContainerIDWithRuntime parses the container ID to get the actual ID string -func ContainerIDWithRuntime(s PodContainerStatus) (string, string) { - cID := s.ContainerID - if cID != "" { - parts := strings.Split(cID, "://") - if len(parts) == 2 { - return parts[1], parts[0] - } - } - return "", "" -} diff --git a/libbeat/common/kubernetes/util.go b/libbeat/common/kubernetes/util.go deleted file mode 100644 index 097accde633..00000000000 --- a/libbeat/common/kubernetes/util.go +++ /dev/null @@ -1,335 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/safemapstr" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-libs/logp" -) - -type HostDiscoveryUtils interface { - GetNamespace() (string, error) - GetPodName() (string, error) - GetMachineID() string -} - -// DiscoverKubernetesNodeParams includes parameters for discovering kubernetes node -type DiscoverKubernetesNodeParams struct { - ConfigHost string - Client kubernetes.Interface - IsInCluster bool - HostUtils HostDiscoveryUtils -} - -// DefaultDiscoveryUtils implements functions of HostDiscoveryUtils interface -type DefaultDiscoveryUtils struct{} - -func GetKubeConfigEnvironmentVariable() string { - envKubeConfig := os.Getenv("KUBECONFIG") - if _, err := os.Stat(envKubeConfig); !os.IsNotExist(err) { - return envKubeConfig - } - return "" -} - -// GetKubernetesClient returns a kubernetes client. If inCluster is true, it returns an -// in cluster configuration based on the secrets mounted in the Pod. If kubeConfig is passed, -// it parses the config file to get the config required to build a client. -func GetKubernetesClient(kubeconfig string, opt KubeClientOptions) (kubernetes.Interface, error) { - if kubeconfig == "" { - kubeconfig = GetKubeConfigEnvironmentVariable() - } - - cfg, err := BuildConfig(kubeconfig) - if err != nil { - return nil, fmt.Errorf("unable to build kube config due to error: %+v", err) - } - cfg.QPS = opt.QPS - cfg.Burst = opt.Burst - client, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, fmt.Errorf("unable to build kubernetes clientset: %+v", err) - } - - return client, nil -} - -// BuildConfig is a helper function that builds configs from a kubeconfig filepath. -// If kubeconfigPath is not passed in we fallback to inClusterConfig. -// If inClusterConfig fails, we fallback to the default config. -// This is a copy of `clientcmd.BuildConfigFromFlags` of `client-go` but without the annoying -// klog messages that are not possible to be disabled. -func BuildConfig(kubeconfigPath string) (*restclient.Config, error) { - if kubeconfigPath == "" { - kubeconfig, err := restclient.InClusterConfig() - if err == nil { - return kubeconfig, nil - } - } - return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}).ClientConfig() -} - -// IsInCluster takes a kubeconfig file path as input and deduces if Beats is running in cluster or not, -// taking into consideration the existence of KUBECONFIG variable -func IsInCluster(kubeconfig string) bool { - if kubeconfig != "" || GetKubeConfigEnvironmentVariable() != "" { - return false - } - return true -} - -// DiscoverKubernetesNode figures out the Kubernetes node to use. -// If host is provided in the config use it directly. -// If it is empty then try -// 1. If beat is deployed in k8s cluster, use hostname of pod as the pod name to query pod metadata for node name. -// 2. If step 1 fails or beat is deployed outside k8s cluster, use machine-id to match against k8s nodes for node name. -// 3. If node cannot be discovered with step 1,2, fallback to NODE_NAME env var as default value. In case it is not set return error. -func DiscoverKubernetesNode(log *logp.Logger, nd *DiscoverKubernetesNodeParams) (string, error) { - ctx := context.TODO() - // Discover node by configuration file (NODE) if set - if nd.ConfigHost != "" { - log.Infof("kubernetes: Using node %s provided in the config", nd.ConfigHost) - return nd.ConfigHost, nil - } - // Discover node by serviceaccount namespace and pod's hostname in case Beats is running in cluster - if nd.IsInCluster { - node, err := discoverInCluster(nd, ctx) - if err == nil { - log.Infof("kubernetes: Node %s discovered by in cluster pod node query", node) - return node, nil - } - log.Debug(err) - } - - // try discover node by machine id - node, err := discoverByMachineId(nd, ctx) - if err == nil { - log.Infof("kubernetes: Node %s discovered by machine-id matching", node) - return node, nil - } - log.Debug(err) - - // fallback to environment variable NODE_NAME - node = os.Getenv("NODE_NAME") - if node != "" { - log.Infof("kubernetes: Node %s discovered by NODE_NAME environment variable", node) - return node, nil - } - - return "", errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") -} - -func discoverInCluster(nd *DiscoverKubernetesNodeParams, ctx context.Context) (node string, errorMsg error) { - ns, err := nd.HostUtils.GetNamespace() - if err != nil { - errorMsg = fmt.Errorf("kubernetes: Couldn't get namespace when beat is in cluster with error: %+v", err.Error()) - return - } - podName, err := nd.HostUtils.GetPodName() - if err != nil { - errorMsg = fmt.Errorf("kubernetes: Couldn't get hostname as beat pod name in cluster with error: %+v", err.Error()) - return - } - pod, err := nd.Client.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - errorMsg = fmt.Errorf("kubernetes: Querying for pod failed with error: %+v", err) - return - } - return pod.Spec.NodeName, nil -} - -func discoverByMachineId(nd *DiscoverKubernetesNodeParams, ctx context.Context) (nodeName string, errorMsg error) { - mid := nd.HostUtils.GetMachineID() - if mid == "" { - errorMsg = errors.New("kubernetes: Couldn't collect info from any of the files in /etc/machine-id /var/lib/dbus/machine-id") - return - } - - nodes, err := nd.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - errorMsg = fmt.Errorf("kubernetes: Querying for nodes failed with error: %+v", err) - return - } - for _, n := range nodes.Items { - if n.Status.NodeInfo.MachineID == mid { - nodeName = n.GetObjectMeta().GetName() - return nodeName, nil - } - } - errorMsg = fmt.Errorf("kubernetes: Couldn't discover node %s", mid) - return -} - -// GetMachineID returns the machine-idadd_kubernetes_metadata/indexers_test.go -// borrowed from machineID of cadvisor. -func (hd *DefaultDiscoveryUtils) GetMachineID() string { - for _, file := range []string{ - "/etc/machine-id", - "/var/lib/dbus/machine-id", - } { - id, err := ioutil.ReadFile(file) - if err == nil { - return strings.TrimSpace(string(id)) - } - } - return "" -} - -// GetNamespace gets namespace from serviceaccount when beat is in cluster. -func (hd *DefaultDiscoveryUtils) GetNamespace() (string, error) { - return InClusterNamespace() -} - -// GetPodName returns the hostname of the pod -func (hd *DefaultDiscoveryUtils) GetPodName() (string, error) { - return os.Hostname() -} - -// InClusterNamespace gets namespace from serviceaccount when beat is in cluster. // code borrowed from client-go with some changes. -func InClusterNamespace() (string, error) { - // get namespace associated with the service account token, if available - data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") - if err != nil { - return "", err - } - return strings.TrimSpace(string(data)), nil -} - -type ContainerInPod struct { - ID string - Runtime string - Spec Container - Status PodContainerStatus -} - -// GetContainersInPod returns all the containers defined in a pod and their statuses. -// It includes init and ephemeral containers. -func GetContainersInPod(pod *Pod) []*ContainerInPod { - var containers []*ContainerInPod - for _, c := range pod.Spec.Containers { - containers = append(containers, &ContainerInPod{Spec: c}) - } - for _, c := range pod.Spec.InitContainers { - containers = append(containers, &ContainerInPod{Spec: c}) - } - for _, c := range pod.Spec.EphemeralContainers { - c := Container(c.EphemeralContainerCommon) - containers = append(containers, &ContainerInPod{Spec: c}) - } - - statuses := make(map[string]*PodContainerStatus) - mapStatuses := func(s []PodContainerStatus) { - for i := range s { - statuses[s[i].Name] = &s[i] - } - } - mapStatuses(pod.Status.ContainerStatuses) - mapStatuses(pod.Status.InitContainerStatuses) - mapStatuses(pod.Status.EphemeralContainerStatuses) - for _, c := range containers { - if s, ok := statuses[c.Spec.Name]; ok { - c.ID, c.Runtime = ContainerIDWithRuntime(*s) - c.Status = *s - } - } - - return containers -} - -// PodAnnotations returns the annotations in a pod -func PodAnnotations(pod *Pod) mapstr.M { - annotations := mapstr.M{} - for k, v := range pod.GetObjectMeta().GetAnnotations() { - safemapstr.Put(annotations, k, v) - } - return annotations -} - -// PodNamespaceAnnotations returns the annotations of the namespace of the pod -func PodNamespaceAnnotations(pod *Pod, watcher Watcher) mapstr.M { - if watcher == nil { - return nil - } - - rawNs, ok, err := watcher.Store().GetByKey(pod.Namespace) - if !ok || err != nil { - return nil - } - - namespace, ok := rawNs.(*Namespace) - if !ok { - return nil - } - - annotations := mapstr.M{} - for k, v := range namespace.GetAnnotations() { - safemapstr.Put(annotations, k, v) - } - return annotations -} - -// PodTerminating returns true if a pod is marked for deletion or is in a phase beyond running. -func PodTerminating(pod *Pod) bool { - if pod.GetObjectMeta().GetDeletionTimestamp() != nil { - return true - } - - switch pod.Status.Phase { - case PodRunning, PodPending: - default: - return true - } - - return false -} - -// PodTerminated returns true if a pod is terminated, this method considers a -// pod as terminated if none of its containers are running (or going to be running). -func PodTerminated(pod *Pod, containers []*ContainerInPod) bool { - // Pod is not marked for termination, so it is not terminated. - if !PodTerminating(pod) { - return false - } - - // If any container is running, the pod is not terminated yet. - for _, container := range containers { - if container.Status.State.Running != nil { - return false - } - } - - return true -} diff --git a/libbeat/common/kubernetes/util_test.go b/libbeat/common/kubernetes/util_test.go deleted file mode 100644 index c651771f89c..00000000000 --- a/libbeat/common/kubernetes/util_test.go +++ /dev/null @@ -1,323 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - k8sfake "k8s.io/client-go/kubernetes/fake" - - core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - "github.com/elastic/elastic-agent-libs/logp" -) - -func TestDiscoverKubernetesNode(t *testing.T) { - client := k8sfake.NewSimpleClientset() - logger := logp.NewLogger("autodiscover.node") - ge := errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") - tests := []struct { - host string - node string - err error - name string - setEnv bool - isInCluster bool - machineid string - podname string - namespace string - init func(*testing.T, kubernetes.Interface) - }{ - { - name: "test value from config", - host: "worker-1", - node: "worker-1", - err: nil, - setEnv: false, - isInCluster: true, - machineid: "", - podname: "", - namespace: "", - }, - { - name: "test value with env var", - host: "", - node: "worker-2", - err: nil, - setEnv: true, - isInCluster: false, - machineid: "", - podname: "", - namespace: "", - }, - { - name: "test value with not incluster, machine ID not retrieved, env var not set", - host: "", - node: "", - err: ge, - setEnv: false, - isInCluster: false, - machineid: "", - podname: "", - namespace: "", - }, - { - name: "test value with inCluster , serviceaccount namespace not found and env var not set", - host: "", - node: "", - err: ge, - setEnv: false, - isInCluster: true, - machineid: "", - podname: "", - namespace: "none", - }, - { - name: "test value with inCluster, pod not found and env var not set", - host: "", - isInCluster: true, - node: "", - err: ge, - setEnv: false, - machineid: "", - podname: "test-pod", - namespace: "default", - }, - { - name: "test value with inCluster, pod found and env var not set", - host: "", - isInCluster: true, - node: "test-node", - err: nil, - setEnv: false, - machineid: "", - podname: "test-pod", - namespace: "default", - init: createResources, - }, - { - name: "test value with inCluster, pod found and env var set", - host: "", - isInCluster: true, - node: "test-node", - err: nil, - setEnv: true, - machineid: "", - podname: "test-pod", - namespace: "default", - init: createResources, - }, - { - name: "test value without inCluster, machine-ID empty and env var not set", - host: "", - isInCluster: false, - node: "", - err: ge, - setEnv: false, - machineid: "", - podname: "", - namespace: "", - }, - { - name: "test value without inCluster, machine-ID set, node not found and env var not set", - host: "", - isInCluster: false, - node: "", - err: ge, - setEnv: false, - machineid: "worker-2", - podname: "", - namespace: "", - }, - { - name: "test value without inCluster, machine-ID set, node found and env var not set", - host: "", - isInCluster: false, - node: "worker-2", - err: nil, - setEnv: false, - machineid: "worker-2", - podname: "", - namespace: "", - init: createResources, - }, - { - name: "test value without inCluster, machine-ID set, node not found and env var set", - host: "", - isInCluster: false, - node: "worker-2", - err: nil, - setEnv: true, - machineid: "worker-2", - podname: "", - namespace: "", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - - if test.setEnv { - - if err := os.Setenv("NODE_NAME", "worker-2"); err != nil { - t.Fatal(err) - } - defer func() { - if err := os.Unsetenv("NODE_NAME"); err != nil { - t.Fatal(err) - } - }() - } - mdu := createMockdu(test.namespace, test.podname, test.machineid) - if test.init != nil { - test.init(t, client) - } - - var nodeName string - var error error - nd := &DiscoverKubernetesNodeParams{ConfigHost: test.host, Client: client, IsInCluster: test.isInCluster, HostUtils: mdu} - nodeName, error = DiscoverKubernetesNode(logger, nd) - - assert.Equal(t, test.node, nodeName) - if error != nil { - assert.Equal(t, test.err.Error(), error.Error()) - } else { - assert.Equal(t, test.err, error) - } - - }) - } -} - -func createPod(client kubernetes.Interface) error { - pod := getPodObject() - - _, err := client.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create k8s pod: %v", err) - } - return nil -} - -func createResources(t *testing.T, client kubernetes.Interface) { - err := createPod(client) - if err != nil { - t.Fatal(err) - } - - err = createNode(client, "worker-2") - if err != nil { - t.Fatal(err) - } - - t.Cleanup(func() { - pod := "test-pod" - err := client.CoreV1().Pods("default").Delete(context.Background(), pod, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("failed to delete k8s pod: %v", err) - } - - err = client.CoreV1().Nodes().Delete(context.Background(), "worker-2", metav1.DeleteOptions{}) - if err != nil { - t.Fatalf("failed to delete k8s node: %v", err) - } - - }) -} - -func createNode(client kubernetes.Interface, name string) error { - node := getNodeObject(name) - - _, err := client.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create k8s node: %v", err) - } - return nil -} - -func getPodObject() *core.Pod { - return &core.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - Labels: map[string]string{ - "app": "demo", - }, - }, - Spec: core.PodSpec{ - NodeName: "test-node", - Containers: []core.Container{ - { - Name: "busybox", - Image: "busybox", - ImagePullPolicy: core.PullIfNotPresent, - Command: []string{ - "sleep", - "3600", - }, - }, - }, - }, - } -} - -func getNodeObject(name string) *core.Node { - return &core.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{ - "name": name, - }, - }, - Spec: core.NodeSpec{}, - Status: core.NodeStatus{NodeInfo: core.NodeSystemInfo{MachineID: name}}, - } -} - -func createMockdu(namespace, podname, machineid string) *mockDiscoveryUtils { - return &mockDiscoveryUtils{namespace: namespace, podname: podname, machineid: machineid} -} - -type mockDiscoveryUtils struct { - namespace string - podname string - machineid string -} - -func (hd *mockDiscoveryUtils) GetMachineID() string { - return hd.machineid -} - -func (hd *mockDiscoveryUtils) GetNamespace() (string, error) { - var error error - if hd.namespace == "none" { - error = errors.New("open /var/run/secrets/kubernetes.io/serviceaccount/namespace: no such file or directory") - } - return hd.namespace, error -} - -func (hd *mockDiscoveryUtils) GetPodName() (string, error) { - return hd.podname, nil -} diff --git a/libbeat/common/kubernetes/watcher.go b/libbeat/common/kubernetes/watcher.go deleted file mode 100644 index 4f57ffec094..00000000000 --- a/libbeat/common/kubernetes/watcher.go +++ /dev/null @@ -1,266 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package kubernetes - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - - "github.com/elastic/elastic-agent-libs/logp" -) - -const ( - add = "add" - update = "update" - delete = "delete" -) - -var ( - accessor = meta.NewAccessor() -) - -// Watcher watches Kubernetes resources events -type Watcher interface { - // Start watching Kubernetes API for new events after resources were listed - Start() error - - // Stop watching Kubernetes API for new events - Stop() - - // AddEventHandler add event handlers for corresponding event type watched - AddEventHandler(ResourceEventHandler) - - // Store returns the store object for the watcher - Store() cache.Store - - // Client returns the kubernetes client object used by the watcher - Client() kubernetes.Interface -} - -// WatchOptions controls watch behaviors -type WatchOptions struct { - // SyncTimeout is a timeout for listing historical resources - SyncTimeout time.Duration - // Node is used for filtering watched resource to given node, use "" for all nodes - Node string - // Namespace is used for filtering watched resource to given namespace, use "" for all namespaces - Namespace string - // IsUpdated allows registering a func that allows the invoker of the Watch to decide what amounts to an update - // vs what does not. - IsUpdated func(old, new interface{}) bool - // HonorReSyncs allows resync events to be requeued on the worker - HonorReSyncs bool -} - -type item struct { - object interface{} - objectRaw interface{} - state string -} - -type watcher struct { - client kubernetes.Interface - informer cache.SharedInformer - store cache.Store - queue workqueue.Interface - ctx context.Context - stop context.CancelFunc - handler ResourceEventHandler - logger *logp.Logger -} - -// NewWatcher initializes the watcher client to provide a events handler for -// resource from the cluster (filtered to the given node) -func NewWatcher(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers) (Watcher, error) { - return NewNamedWatcher("", client, resource, opts, indexers) -} - -// NewNamedWatcher initializes the watcher client to provide an events handler for -// resource from the cluster (filtered to the given node) and also allows to name the k8s -// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue -// metrics, if it is empty, its metrics will not be logged by the k8s client. -func NewNamedWatcher(name string, client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers) (Watcher, error) { - var store cache.Store - var queue workqueue.Interface - - informer, _, err := NewInformer(client, resource, opts, indexers) - if err != nil { - return nil, err - } - - store = informer.GetStore() - queue = workqueue.NewNamed(name) - - if opts.IsUpdated == nil { - opts.IsUpdated = func(o, n interface{}) bool { - old, _ := accessor.ResourceVersion(o.(runtime.Object)) - new, _ := accessor.ResourceVersion(n.(runtime.Object)) - - // Only enqueue changes that have a different resource versions to avoid processing resyncs. - if old != new { - return true - } - return false - } - } - - ctx, cancel := context.WithCancel(context.TODO()) - w := &watcher{ - client: client, - informer: informer, - store: store, - queue: queue, - ctx: ctx, - stop: cancel, - logger: logp.NewLogger("kubernetes"), - handler: NoOpEventHandlerFuncs{}, - } - - w.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(o interface{}) { - w.enqueue(o, add) - }, - DeleteFunc: func(o interface{}) { - w.enqueue(o, delete) - }, - UpdateFunc: func(o, n interface{}) { - if opts.IsUpdated(o, n) { - w.enqueue(n, update) - } else if opts.HonorReSyncs { - // HonorReSyncs ensure that at the time when the kubernetes client does a "resync", i.e, a full list of all - // objects we make sure that autodiscover processes them. Why is this necessary? An effective control loop works - // based on two state changes, a list and a watch. A watch is triggered each time the state of the system changes. - // However, there is no guarantee that all events from a watch are processed by the receiver. To ensure that missed events - // are properly handled, a period re-list is done to ensure that every state within the system is effectively handled. - // In this case, we are making sure that we are enqueueing an "add" event because, an runner that is already in Running - // state should just be deduped by autodiscover and not stop/started periodically as would be the case with an update. - w.enqueue(n, add) - } - }, - }) - - return w, nil -} - -// AddEventHandler adds a resource handler to process each request that is coming into the watcher -func (w *watcher) AddEventHandler(h ResourceEventHandler) { - w.handler = h -} - -// Store returns the store object for the resource that is being watched -func (w *watcher) Store() cache.Store { - return w.store -} - -// Client returns the kubernetes client object used by the watcher -func (w *watcher) Client() kubernetes.Interface { - return w.client -} - -// Start watching pods -func (w *watcher) Start() error { - go w.informer.Run(w.ctx.Done()) - - if !cache.WaitForCacheSync(w.ctx.Done(), w.informer.HasSynced) { - return fmt.Errorf("kubernetes informer unable to sync cache") - } - - w.logger.Debugf("cache sync done") - - //TODO: Do we run parallel workers for this? It is useful when we run metricbeat as one instance per cluster? - - // Wrap the process function with wait.Until so that if the controller crashes, it starts up again after a second. - go wait.Until(func() { - for w.process(w.ctx) { - } - }, time.Second*1, w.ctx.Done()) - - return nil -} - -func (w *watcher) Stop() { - w.queue.ShutDown() - w.stop() -} - -// enqueue takes the most recent object that was received, figures out the namespace/name of the object -// and adds it to the work queue for processing. -func (w *watcher) enqueue(obj interface{}, state string) { - // DeletionHandlingMetaNamespaceKeyFunc that we get a key only if the resource's state is not Unknown. - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - return - } - if deleted, ok := obj.(cache.DeletedFinalStateUnknown); ok { - w.logger.Debugf("Enqueued DeletedFinalStateUnknown contained object: %+v", deleted.Obj) - obj = deleted.Obj - } - w.queue.Add(&item{key, obj, state}) -} - -// process gets the top of the work queue and processes the object that is received. -func (w *watcher) process(ctx context.Context) bool { - obj, quit := w.queue.Get() - if quit { - return false - } - defer w.queue.Done(obj) - - var entry *item - var ok bool - if entry, ok = obj.(*item); !ok { - utilruntime.HandleError(fmt.Errorf("expected *item in workqueue but got %#v", obj)) - return true - } - - key := entry.object.(string) - - o, exists, err := w.store.GetByKey(key) - if err != nil { - utilruntime.HandleError(fmt.Errorf("getting object %#v from cache: %w", obj, err)) - return true - } - if !exists { - if entry.state == delete { - w.logger.Debugf("Object %+v was not found in the store, deleting anyway!", key) - // delete anyway in order to clean states - w.handler.OnDelete(entry.objectRaw) - } - return true - } - - switch entry.state { - case add: - w.handler.OnAdd(o) - case update: - w.handler.OnUpdate(o) - case delete: - w.handler.OnDelete(o) - } - - return true -} diff --git a/libbeat/keystore/config.go b/libbeat/keystore/config.go deleted file mode 100644 index 16de2cd4a52..00000000000 --- a/libbeat/keystore/config.go +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -// Config Define keystore configurable options -type Config struct { - Path string `config:"path"` -} - -var defaultConfig = Config{ - Path: "", -} diff --git a/libbeat/keystore/file_keystore.go b/libbeat/keystore/file_keystore.go deleted file mode 100644 index 44a13f40dc8..00000000000 --- a/libbeat/keystore/file_keystore.go +++ /dev/null @@ -1,457 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/sha512" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "sync" - - "golang.org/x/crypto/pbkdf2" - - "github.com/elastic/beats/v7/libbeat/common" - c "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/file" -) - -const ( - filePermission = 0600 - - // Encryption Related constants - iVLength = 12 - saltLength = 64 - iterationsCount = 10000 - keyLength = 32 -) - -// Version of the keystore format, will be added at the beginning of the file. -var version = []byte("v1") - -// Packager defines a keystore that we can read the raw bytes and be packaged in an artifact. -type Packager interface { - Package() ([]byte, error) - ConfiguredPath() string -} - -// FileKeystore Allows to store key / secrets pair securely into an encrypted local file. -type FileKeystore struct { - sync.RWMutex - Path string - secrets map[string]serializableSecureString - dirty bool - password *SecureString -} - -// Allow the original SecureString type to be correctly serialized to json. -type serializableSecureString struct { - *SecureString - Value []byte `json:"value"` -} - -// Factory Create the right keystore with the configured options. -func Factory(cfg *c.C, defaultPath string) (Keystore, error) { - config := defaultConfig - - if cfg == nil { - cfg = c.NewConfig() - } - err := cfg.Unpack(&config) - - if err != nil { - return nil, fmt.Errorf("could not read keystore configuration, err: %v", err) - } - - if config.Path == "" { - config.Path = defaultPath - } - - keystore, err := NewFileKeystore(config.Path) - return keystore, err -} - -// NewFileKeystore returns an new File based keystore or an error, currently users cannot set their -// own password on the keystore, the default password will be an empty string. When the keystore -// is initialized the secrets are automatically loaded into memory. -func NewFileKeystore(keystoreFile string) (Keystore, error) { - return NewFileKeystoreWithPassword(keystoreFile, NewSecureString([]byte(""))) -} - -// NewFileKeystoreWithPassword return a new File based keystore or an error, allow to define what -// password to use to create the keystore. -func NewFileKeystoreWithPassword(keystoreFile string, password *SecureString) (Keystore, error) { - keystore := FileKeystore{ - Path: keystoreFile, - dirty: false, - password: password, - secrets: make(map[string]serializableSecureString), - } - - err := keystore.load() - if err != nil { - return nil, err - } - - return &keystore, nil -} - -// Retrieve return a SecureString instance that will contains both the key and the secret. -func (k *FileKeystore) Retrieve(key string) (*SecureString, error) { - k.RLock() - defer k.RUnlock() - - secret, ok := k.secrets[key] - if !ok { - return nil, ErrKeyDoesntExists - } - return NewSecureString(secret.Value), nil -} - -// Store add the key pair to the secret store and mark the store as dirty. -func (k *FileKeystore) Store(key string, value []byte) error { - k.Lock() - defer k.Unlock() - - k.secrets[key] = serializableSecureString{Value: value} - k.dirty = true - return nil -} - -// Delete an existing key from the store and mark the store as dirty. -func (k *FileKeystore) Delete(key string) error { - k.Lock() - defer k.Unlock() - - delete(k.secrets, key) - k.dirty = true - return nil -} - -// Save persists the in memory data to disk if needed. -func (k *FileKeystore) Save() error { - k.Lock() - err := k.doSave(true) - k.Unlock() - return err -} - -// List return the availables keys. -func (k *FileKeystore) List() ([]string, error) { - k.RLock() - defer k.RUnlock() - - keys := make([]string, 0, len(k.secrets)) - for key := range k.secrets { - keys = append(keys, key) - } - - return keys, nil -} - -// GetConfig returns config.C representation of the key / secret pair to be merged with other -// loaded configuration. -func (k *FileKeystore) GetConfig() (*c.C, error) { - k.RLock() - defer k.RUnlock() - - configHash := make(map[string]interface{}) - for key, secret := range k.secrets { - configHash[key] = string(secret.Value) - } - - return c.NewConfigFrom(configHash) -} - -// Create create an empty keystore, if the store already exist we will return an error. -func (k *FileKeystore) Create(override bool) error { - k.Lock() - k.secrets = make(map[string]serializableSecureString) - k.dirty = true - err := k.doSave(override) - k.Unlock() - return err -} - -// IsPersisted return if the keystore is physically persisted on disk. -func (k *FileKeystore) IsPersisted() bool { - k.Lock() - defer k.Unlock() - - // We just check if the file is present on disk, we don't need to do any validation - // for a file based keystore, since all the keys will be fetched when we initialize the object - // if the file is invalid it will already fails. Creating a new FileKeystore will raise - // any errors concerning the permissions - f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission) - if err != nil { - return false - } - f.Close() - return true -} - -// doSave lock/unlocking of the resource need to be done by the caller. -func (k *FileKeystore) doSave(override bool) error { - if !k.dirty { - return nil - } - - temporaryPath := fmt.Sprintf("%s.tmp", k.Path) - - w := new(bytes.Buffer) - jsonEncoder := json.NewEncoder(w) - if err := jsonEncoder.Encode(k.secrets); err != nil { - return fmt.Errorf("cannot serialize the keystore before saving it to disk: %v", err) - } - - encrypted, err := k.encrypt(w) - if err != nil { - return fmt.Errorf("cannot encrypt the keystore: %v", err) - } - - flags := os.O_RDWR | os.O_CREATE - if override { - flags |= os.O_TRUNC - } else { - flags |= os.O_EXCL - } - - f, err := os.OpenFile(temporaryPath, flags, filePermission) - if err != nil { - return fmt.Errorf("cannot open file to save the keystore to '%s', error: %s", k.Path, err) - } - - // Writing files can and allowed to go wrong here. - // Where it is important to write to disk we use SafeFileRotate or safeWriteFile. - // As this function only makes a checkpoint, the keystore is going to be in a consistent - // state eventually. - _, _ = f.Write(version) - base64Encoder := base64.NewEncoder(base64.StdEncoding, f) - _, _ = io.Copy(base64Encoder, encrypted) - base64Encoder.Close() - _ = f.Sync() - f.Close() - - err = file.SafeFileRotate(k.Path, temporaryPath) - if err != nil { - os.Remove(temporaryPath) - return fmt.Errorf("cannot replace the existing keystore, with the new keystore file at '%s', error: %s", k.Path, err) - } - os.Remove(temporaryPath) - - k.dirty = false - return nil -} - -func (k *FileKeystore) loadRaw() ([]byte, error) { - f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - defer f.Close() - - if common.IsStrictPerms() { - if err := k.checkPermissions(k.Path); err != nil { - return nil, err - } - } - - raw, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - v := raw[0:len(version)] - if !bytes.Equal(v, version) { - return nil, fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v) - } - - if len(raw) <= len(version) { - return nil, fmt.Errorf("corrupt or empty keystore") - } - - return raw, nil -} - -func (k *FileKeystore) load() error { - k.Lock() - defer k.Unlock() - - raw, err := k.loadRaw() - if err != nil { - return err - } - - if len(raw) == 0 { - return nil - } - - base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(raw[len(version):])) - plaintext, err := k.decrypt(base64Decoder) - if err != nil { - return fmt.Errorf("could not decrypt the keystore: %v", err) - } - jsonDecoder := json.NewDecoder(plaintext) - return jsonDecoder.Decode(&k.secrets) -} - -// Encrypt the data payload using a derived keys and the AES-256-GCM algorithm. -func (k *FileKeystore) encrypt(reader io.Reader) (io.Reader, error) { - // randomly generate the salt and the initialization vector, this information will be saved - // on disk in the file as part of the header - iv, err := common.RandomBytes(iVLength) - - if err != nil { - return nil, err - } - - salt, err := common.RandomBytes(saltLength) - if err != nil { - return nil, err - } - - // Stretch the user provided key - password, _ := k.password.Get() - passwordBytes := k.hashPassword(password, salt) - - // Select AES-256: because len(passwordBytes) == 32 bytes - block, err := aes.NewCipher(passwordBytes) - if err != nil { - return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %s", err) - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %s", err) - } - - data, err := ioutil.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("could not read unencrypted data, error: %s", err) - } - - encodedBytes := aesgcm.Seal(nil, iv, data, nil) - - // Generate the payload with all the additional information required to decrypt the - // output format of the document: VERSION|SALT|IV|PAYLOAD - buf := bytes.NewBuffer(salt) - buf.Write(iv) - buf.Write(encodedBytes) - - return buf, nil -} - -// should receive an io.reader... -func (k *FileKeystore) decrypt(reader io.Reader) (io.Reader, error) { - data, err := ioutil.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("could not read all the data from the encrypted file, error: %s", err) - } - - if len(data) < saltLength+iVLength+1 { - return nil, fmt.Errorf("missing information in the file for decrypting the keystore") - } - - // extract the necessary information to decrypt the data from the data payload - salt := data[0:saltLength] - iv := data[saltLength : saltLength+iVLength] - encodedBytes := data[saltLength+iVLength:] - - password, _ := k.password.Get() - passwordBytes := k.hashPassword(password, salt) - - block, err := aes.NewCipher(passwordBytes) - if err != nil { - return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %s", err) - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %s", err) - } - - decodedBytes, err := aesgcm.Open(nil, iv, encodedBytes, nil) - if err != nil { - return nil, fmt.Errorf("could not decrypt keystore data: %s", err) - } - - return bytes.NewReader(decodedBytes), nil -} - -// checkPermission enforces permission on the keystore file itself, the file should have strict -// permission (0600) and the keystore should refuses to start if its not the case. -func (k *FileKeystore) checkPermissions(f string) error { - if runtime.GOOS == "windows" { - return nil - } - - info, err := file.Stat(f) - if err != nil { - return err - } - - euid := os.Geteuid() - fileUID, _ := info.UID() - perm := info.Mode().Perm() - - if fileUID != 0 && euid != fileUID { - return fmt.Errorf(`config file ("%v") must be owned by the user identifier `+ - `(uid=%v) or root`, f, euid) - } - - // Test if group or other have write permissions. - if perm != filePermission { - nameAbs, err := filepath.Abs(f) - if err != nil { - nameAbs = f - } - return fmt.Errorf(`file ("%v") can only be writable and readable by the `+ - `owner but the permissions are "%v" (to fix the permissions use: `+ - `'chmod go-wrx %v')`, - f, perm, nameAbs) - } - - return nil -} - -// Package returns the bytes of the encrypted keystore. -func (k *FileKeystore) Package() ([]byte, error) { - k.Lock() - defer k.Unlock() - return k.loadRaw() -} - -// ConfiguredPath returns the path to the keystore. -func (k *FileKeystore) ConfiguredPath() string { - return k.Path -} - -func (k *FileKeystore) hashPassword(password, salt []byte) []byte { - return pbkdf2.Key(password, salt, iterationsCount, keyLength, sha512.New) -} diff --git a/libbeat/keystore/file_keystore_test.go b/libbeat/keystore/file_keystore_test.go deleted file mode 100644 index fbfaea83da9..00000000000 --- a/libbeat/keystore/file_keystore_test.go +++ /dev/null @@ -1,343 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/docker/docker/pkg/ioutils" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/common" -) - -var keyValue = "output.elasticsearch.password" -var secretValue = []byte("secret") - -func TestCanCreateAKeyStore(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore, err := NewFileKeystore(path) - assert.NoError(t, err) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - assert.Nil(t, writableKeystore.Store(keyValue, secretValue)) - assert.Nil(t, writableKeystore.Save()) -} - -func TestCanReadAnExistingKeyStoreWithEmptyString(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - CreateAnExistingKeystore(path) - - keystoreRead, err := NewFileKeystore(path) - assert.NoError(t, err) - - secure, err := keystoreRead.Retrieve(keyValue) - assert.NoError(t, err) - - v, err := secure.Get() - assert.NoError(t, err) - assert.Equal(t, v, secretValue) -} - -func TestCanDeleteAKeyFromTheStoreAndPersistChanges(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - CreateAnExistingKeystore(path) - - keyStore, _ := NewFileKeystore(path) - _, err := keyStore.Retrieve(keyValue) - assert.NoError(t, err) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - writableKeystore.Delete(keyValue) - _, err = keyStore.Retrieve(keyValue) - assert.Error(t, err) - - _ = writableKeystore.Save() - newKeystore, err := NewFileKeystore(path) - _, err = newKeystore.Retrieve(keyValue) - assert.Error(t, err) -} - -func TestFilePermissionOnCreate(t *testing.T) { - // Skip check on windows - if runtime.GOOS == "windows" { - t.Skip("Permission check is not running on windows") - } - if !common.IsStrictPerms() { - t.Skip("Skipping test because strict.perms is disabled") - } - - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - CreateAnExistingKeystore(path) - - stats, err := os.Stat(path) - assert.NoError(t, err) - permissions := stats.Mode().Perm() - if permissions != 0600 { - t.Fatalf("Expecting the file what only readable/writable by the owner, permission found: %v", permissions) - } -} - -func TestFilePermissionOnUpdate(t *testing.T) { - // Skip check on windows - if runtime.GOOS == "windows" { - t.Skip("Permission check is not running on windows") - } - if !common.IsStrictPerms() { - t.Skip("Skipping test because strict.perms is disabled") - } - - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore := CreateAnExistingKeystore(path) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - err = writableKeystore.Store("newkey", []byte("newsecret")) - assert.NoError(t, err) - err = writableKeystore.Save() - assert.NoError(t, err) - stats, err := os.Stat(path) - assert.NoError(t, err) - permissions := stats.Mode().Perm() - if permissions != 0600 { - t.Fatalf("Expecting the file what only readable/writable by the owner, permission found: %v", permissions) - } -} - -func TestFilePermissionOnLoadWhenStrictIsOn(t *testing.T) { - // Skip check on windows - if runtime.GOOS == "windows" { - t.Skip("Permission check is not running on windows") - } - - if !common.IsStrictPerms() { - t.Skip("Skipping test because strict.perms is disabled") - } - - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - // Create a world readable keystore file - fd, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) - assert.NoError(t, err) - fd.WriteString("bad permission") - assert.NoError(t, fd.Close()) - _, err = NewFileKeystore(path) - assert.Error(t, err) -} - -func TestReturnsUsedKeysInTheStore(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore := CreateAnExistingKeystore(path) - - listingKeystore, err := AsListingKeystore(keyStore) - assert.NoError(t, err) - - keys, err := listingKeystore.List() - - assert.NoError(t, err) - assert.Equal(t, len(keys), 1) - assert.Equal(t, keys[0], keyValue) -} - -func TestCannotDecryptKeyStoreWithWrongPassword(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString([]byte("password"))) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - writableKeystore.Store("hello", []byte("world")) - writableKeystore.Save() - - _, err = NewFileKeystoreWithPassword(path, NewSecureString([]byte("wrongpassword"))) - if assert.Error(t, err, "should fail to decrypt the keystore") { - m := `could not decrypt the keystore: could not decrypt keystore data: ` + - `cipher: message authentication failed` - assert.Equal(t, err, fmt.Errorf(m)) - } -} - -func TestUserDefinedPasswordUTF8(t *testing.T) { - createAndReadKeystoreWithPassword(t, []byte("mysecret¥¥password")) -} - -func TestUserDefinedPasswordASCII(t *testing.T) { - createAndReadKeystoreWithPassword(t, []byte("mysecret")) -} - -func TestSecretWithUTF8EncodedSecret(t *testing.T) { - content := []byte("ありがとうございます") // translation: thank you - createAndReadKeystoreSecret(t, []byte("mysuperpassword"), "mykey", content) -} - -func TestSecretWithASCIIEncodedSecret(t *testing.T) { - content := []byte("good news everyone") // translation: thank you - createAndReadKeystoreSecret(t, []byte("mysuperpassword"), "mykey", content) -} - -func TestGetConfig(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore := CreateAnExistingKeystore(path) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - // Add a bit more data of different type - writableKeystore.Store("super.nested", []byte("hello")) - writableKeystore.Save() - - cfg, err := keyStore.GetConfig() - assert.NotNil(t, cfg) - assert.NoError(t, err) - - secret, err := cfg.String("output.elasticsearch.password", 0) - assert.NoError(t, err) - assert.Equal(t, secret, "secret") - - port, err := cfg.String("super.nested", 0) - assert.Equal(t, port, "hello") -} - -func TestShouldRaiseAndErrorWhenVersionDontMatch(t *testing.T) { - temporaryPath := GetTemporaryKeystoreFile() - defer os.Remove(temporaryPath) - - badVersion := `v2D/EQwnDNO7yZsjsRFVWGgbkZudhPxVhBkaQAVud66+tK4HRdfPrNrNNgSmhioDGrQ0z/VZpvbw68gb0G - G2QHxlP5s4HGRU/GQge3Nsnx0+kDIcb/37gPN1D1TOPHSiRrzzPn2vInmgaLUfEgBgoa9tuXLZEKdh3JPh/q` - - f, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_WRONLY, 0600) - assert.NoError(t, err) - f.WriteString(badVersion) - err = f.Close() - assert.NoError(t, err) - - _, err = NewFileKeystoreWithPassword(temporaryPath, NewSecureString([]byte(""))) - if assert.Error(t, err, "Expect version check error") { - assert.Equal(t, err, fmt.Errorf("keystore format doesn't match expected version: 'v1' got 'v2'")) - } -} - -func TestMissingEncryptedBlock(t *testing.T) { - temporaryPath := GetTemporaryKeystoreFile() - defer os.Remove(temporaryPath) - - badVersion := "v1" - - f, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_WRONLY, 0600) - assert.NoError(t, err) - f.WriteString(badVersion) - err = f.Close() - assert.NoError(t, err) - - _, err = NewFileKeystoreWithPassword(temporaryPath, NewSecureString([]byte(""))) - if assert.Error(t, err) { - assert.Equal(t, err, fmt.Errorf("corrupt or empty keystore")) - } -} - -func createAndReadKeystoreSecret(t *testing.T, password []byte, key string, value []byte) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) - assert.NoError(t, err) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - writableKeystore.Store(key, value) - writableKeystore.Save() - - newStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) - s, _ := newStore.Retrieve(key) - v, _ := s.Get() - assert.Equal(t, v, value) -} - -func createAndReadKeystoreWithPassword(t *testing.T, password []byte) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) - assert.NoError(t, err) - - writableKeystore, err := AsWritableKeystore(keyStore) - assert.NoError(t, err) - - writableKeystore.Store("hello", []byte("world")) - writableKeystore.Save() - - newStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) - s, _ := newStore.Retrieve("hello") - v, _ := s.Get() - - assert.Equal(t, v, []byte("world")) -} - -// CreateAnExistingKeystore creates a keystore with an existing key -/// `output.elasticsearch.password` with the value `secret`. -func CreateAnExistingKeystore(path string) Keystore { - keyStore, err := NewFileKeystore(path) - // Fail fast in the test suite - if err != nil { - panic(err) - } - - writableKeystore, err := AsWritableKeystore(keyStore) - if err != nil { - panic(err) - } - - writableKeystore.Store(keyValue, secretValue) - writableKeystore.Save() - return keyStore -} - -// GetTemporaryKeystoreFile create a temporary file on disk to save the keystore. -func GetTemporaryKeystoreFile() string { - path, err := ioutils.TempDir("", "testing") - if err != nil { - panic(err) - } - return filepath.Join(path, "keystore") -} diff --git a/libbeat/keystore/keystore.go b/libbeat/keystore/keystore.go deleted file mode 100644 index d929af7c632..00000000000 --- a/libbeat/keystore/keystore.go +++ /dev/null @@ -1,123 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -import ( - "errors" - - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/go-ucfg" - "github.com/elastic/go-ucfg/parse" -) - -var ( - // ErrAlreadyExists is returned when the file already exist at the location. - ErrAlreadyExists = errors.New("cannot create a new keystore a valid keystore already exist at the location") - - // ErrKeyDoesntExists is returned when the key doesn't exist in the store - ErrKeyDoesntExists = errors.New("cannot retrieve the key") - - // ErrNotWritable is returned when the keystore is not writable - ErrNotWritable = errors.New("the configured keystore is not writable") - - // ErrNotWritable is returned when the keystore is not writable - ErrNotListing = errors.New("the configured keystore is not listing") -) - -// Keystore implement a way to securely saves and retrieves secrets to be used in the configuration -// Currently all credentials are loaded upfront and are not lazy retrieved, we will eventually move -// to that concept, so we can deal with tokens that has a limited duration or can be revoked by a -// remote keystore. -type Keystore interface { - // Retrieve returns a SecureString instance of the searched key or an error. - Retrieve(key string) (*SecureString, error) - - // GetConfig returns the key value pair in the config format to be merged with other configuration. - GetConfig() (*config.C, error) - - // IsPersisted check if the current keystore is persisted. - IsPersisted() bool -} - -type WritableKeystore interface { - // Store add keys to the keystore, wont be persisted until we save. - Store(key string, secret []byte) error - - // Delete removes a specific key from the keystore. - Delete(key string) error - - // Create Allow to create an empty keystore. - Create(override bool) error - - // Save persist the changes to the keystore. - Save() error -} - -type ListingKeystore interface { - // List returns the list of keys in the keystore, return an empty list if none is found. - List() ([]string, error) -} - -// Provider for keystore -type Provider interface { - GetKeystore(event bus.Event) Keystore -} - -// ResolverWrap wrap a config resolver around an existing keystore. -func ResolverWrap(keystore Keystore) func(string) (string, parse.Config, error) { - return func(keyName string) (string, parse.Config, error) { - key, err := keystore.Retrieve(keyName) - - if err != nil { - // If we cannot find the key, its a non fatal error - // and we pass to other resolver. - if err == ErrKeyDoesntExists { - return "", parse.DefaultConfig, ucfg.ErrMissing - } - return "", parse.DefaultConfig, err - } - - v, err := key.Get() - if err != nil { - return "", parse.DefaultConfig, err - } - - return string(v), parse.DefaultConfig, nil - } -} - -// AsWritableKeystore casts a keystore to WritableKeystore, returning an ErrNotWritable error if the given keystore does not implement -// WritableKeystore interface -func AsWritableKeystore(store Keystore) (WritableKeystore, error) { - w, ok := store.(WritableKeystore) - if !ok { - return nil, ErrNotWritable - } - return w, nil -} - -// AsListingKeystore casts a keystore to ListingKeystore, returning an ErrNotListing error if the given keystore does not implement -// ListingKeystore interface -func AsListingKeystore(store Keystore) (ListingKeystore, error) { - w, ok := store.(ListingKeystore) - if !ok { - return nil, ErrNotListing - } - return w, nil -} diff --git a/libbeat/keystore/keystore_test.go b/libbeat/keystore/keystore_test.go deleted file mode 100644 index 36c96f3fe96..00000000000 --- a/libbeat/keystore/keystore_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - - ucfg "github.com/elastic/go-ucfg" - "github.com/elastic/go-ucfg/parse" -) - -func TestResolverWhenTheKeyDoesntExist(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keystore := CreateAnExistingKeystore(path) - - resolver := ResolverWrap(keystore) - _, _, err := resolver("donotexist") - assert.Equal(t, err, ucfg.ErrMissing) -} - -func TestResolverWhenTheKeyExist(t *testing.T) { - path := GetTemporaryKeystoreFile() - defer os.Remove(path) - - keystore := CreateAnExistingKeystore(path) - - resolver := ResolverWrap(keystore) - v, pCfg, err := resolver("output.elasticsearch.password") - assert.NoError(t, err) - assert.Equal(t, pCfg, parse.DefaultConfig) - assert.Equal(t, v, "secret") -} diff --git a/libbeat/keystore/secure_string.go b/libbeat/keystore/secure_string.go deleted file mode 100644 index d6f2ce9b55c..00000000000 --- a/libbeat/keystore/secure_string.go +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -// SecureString Initial implementation for a SecureString representation in -// beats, currently we keep the password into a Bytes array, we need to implement a way -// to safely clean that array. -// -// Investigate memguard: https://github.com/awnumar/memguard -type SecureString struct { - value []byte -} - -// NewSecureString return a struct representing a secrets string. -func NewSecureString(value []byte) *SecureString { - return &SecureString{ - value: value, - } -} - -// Get returns the byte value of the secret, or an error if we cannot return it. -func (s *SecureString) Get() ([]byte, error) { - return s.value, nil -} - -// String custom string implementation to make sure we don't bleed this struct into a string. -func (s SecureString) String() string { - return "" -} - -// GoString implements the GoStringer interface to hide the secret value. -func (s SecureString) GoString() string { - return s.String() -} diff --git a/libbeat/keystore/secure_string_test.go b/libbeat/keystore/secure_string_test.go deleted file mode 100644 index 5ce407ecc89..00000000000 --- a/libbeat/keystore/secure_string_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package keystore - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -var secret = []byte("mysecret") - -func TestGet(t *testing.T) { - s := NewSecureString(secret) - v, err := s.Get() - assert.Equal(t, secret, v) - assert.NoError(t, err) -} - -func TestStringMarshalingS(t *testing.T) { - s := NewSecureString(secret) - v := fmt.Sprintf("%s", s) - - assert.Equal(t, v, "") -} - -func TestStringMarshalingF(t *testing.T) { - s := NewSecureString(secret) - v := fmt.Sprintf("%v", s) - - assert.Equal(t, v, "") -} - -func TestStringGoStringerMarshaling(t *testing.T) { - s := NewSecureString(secret) - v := fmt.Sprintf("%#v", s) - - assert.Equal(t, v, "") -} diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 22ac63be041..41fa8b13ba3 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -31,11 +31,11 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/libbeat/metric/system/cgroup" "github.com/elastic/beats/v7/libbeat/metric/system/resolve" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/actions" + "github.com/elastic/elastic-agent-autodiscover/docker" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go index 1e983d80851..541d31291e8 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go @@ -29,10 +29,10 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/libbeat/processors" dockertest "github.com/elastic/beats/v7/libbeat/tests/docker" "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index a473e6fc9cb..c910744beb8 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -30,10 +30,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/libbeat/metric/system/cgroup" "github.com/elastic/beats/v7/libbeat/metric/system/resolve" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index ddbf9b88796..082fdb72f7d 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -23,7 +23,7 @@ package add_docker_metadata import ( "time" - "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/elastic-agent-autodiscover/docker" ) // Config for docker processor. diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index ec014541693..0998a275ea4 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/processors/add_kubernetes_metadata/indexers.go b/libbeat/processors/add_kubernetes_metadata/indexers.go index b3dbba40698..f60a43650bd 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers.go @@ -20,12 +20,11 @@ package add_kubernetes_metadata import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) const ( diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index fc57fd60fb2..63f073479fd 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -21,7 +21,8 @@ import ( "fmt" "testing" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" @@ -29,8 +30,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" ) var addResourceMetadata = metadata.GetDefaultResourceMetadataConfig() diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 2bd7caefde1..0bc478f8abc 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -29,9 +29,9 @@ import ( k8sclient "k8s.io/client-go/kubernetes" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index 87919c3fd9c..2c81e2557b3 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -38,7 +38,7 @@ import ( "github.com/docker/docker/client" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/elastic-agent-autodiscover/docker" ) const ( diff --git a/libbeat/tests/docker/docker.go b/libbeat/tests/docker/docker.go index fa53c832cea..1b40bef0d80 100644 --- a/libbeat/tests/docker/docker.go +++ b/libbeat/tests/docker/docker.go @@ -24,11 +24,11 @@ import ( "github.com/pkg/errors" + "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/client" - - "github.com/elastic/beats/v7/libbeat/common/docker" ) // Client for Docker diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index d01d398c674..4c474d36b16 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -22,9 +22,9 @@ import ( "io/ioutil" "github.com/elastic/beats/v7/libbeat/autodiscover" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go index 8192c772dac..62243867958 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 5983ef00710..a5ad68e3cd4 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-ucfg" @@ -29,7 +30,6 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/metricbeat/mb" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index ebce8769f57..27db4f54d98 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -26,9 +26,9 @@ import ( "github.com/docker/docker/pkg/ioutils" "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/container/data.go b/metricbeat/module/docker/container/data.go index 3047ddd6f84..378ee12f39f 100644 --- a/metricbeat/module/docker/container/data.go +++ b/metricbeat/module/docker/container/data.go @@ -23,9 +23,9 @@ import ( "github.com/docker/docker/api/types" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/docker.go b/metricbeat/module/docker/docker.go index 3d0856477bc..17ec3ba4b21 100644 --- a/metricbeat/module/docker/docker.go +++ b/metricbeat/module/docker/docker.go @@ -31,9 +31,9 @@ import ( "github.com/docker/docker/client" "github.com/docker/go-connections/tlsconfig" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/elastic-agent-autodiscover/docker" ) // HostParser is a TCP host parser function for docker tcp host addresses diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index 28b916ad523..60b1033eaca 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -32,9 +32,9 @@ import ( "github.com/docker/docker/client" "github.com/elastic/beats/v7/auditbeat/core" - "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/elastic-agent-autodiscover/docker" ) func TestData(t *testing.T) { diff --git a/metricbeat/module/docker/helper.go b/metricbeat/module/docker/helper.go index 2af4f7b7228..0e272039715 100644 --- a/metricbeat/module/docker/helper.go +++ b/metricbeat/module/docker/helper.go @@ -20,7 +20,7 @@ package docker import ( "github.com/docker/docker/api/types" - helpers "github.com/elastic/beats/v7/libbeat/common/docker" + helpers "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/helper_test.go b/metricbeat/module/docker/helper_test.go index a7e750e3fae..0c9238f7506 100644 --- a/metricbeat/module/docker/helper_test.go +++ b/metricbeat/module/docker/helper_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - helpers "github.com/elastic/beats/v7/libbeat/common/docker" + helpers "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/image/data.go b/metricbeat/module/docker/image/data.go index f2b59979aee..2b15745440a 100644 --- a/metricbeat/module/docker/image/data.go +++ b/metricbeat/module/docker/image/data.go @@ -23,7 +23,7 @@ import ( "github.com/docker/docker/api/types" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/event/config.go b/metricbeat/module/kubernetes/event/config.go index be9b0c21a15..cc62c3395c7 100644 --- a/metricbeat/module/kubernetes/event/config.go +++ b/metricbeat/module/kubernetes/event/config.go @@ -20,7 +20,7 @@ package event import ( "time" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) type kubeEventsConfig struct { diff --git a/metricbeat/module/kubernetes/event/event.go b/metricbeat/module/kubernetes/event/event.go index dac99a68f13..88e07353538 100644 --- a/metricbeat/module/kubernetes/event/event.go +++ b/metricbeat/module/kubernetes/event/event.go @@ -22,9 +22,9 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/metricbeat/module/kubernetes/node/node.go b/metricbeat/module/kubernetes/node/node.go index 80bd60d67fa..3dd38d910b1 100644 --- a/metricbeat/module/kubernetes/node/node.go +++ b/metricbeat/module/kubernetes/node/node.go @@ -20,12 +20,12 @@ package node import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/pod/pod.go b/metricbeat/module/kubernetes/pod/pod.go index eae281feb9d..c936b6017a8 100644 --- a/metricbeat/module/kubernetes/pod/pod.go +++ b/metricbeat/module/kubernetes/pod/pod.go @@ -20,12 +20,12 @@ package pod import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go b/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go index 49d191580fd..f8e83e5ed22 100644 --- a/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go +++ b/metricbeat/module/kubernetes/state_cronjob/state_cronjob.go @@ -20,8 +20,8 @@ package state_cronjob import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" diff --git a/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go b/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go index 8d80fc3f176..541a09ca540 100644 --- a/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go +++ b/metricbeat/module/kubernetes/state_daemonset/state_daemonset.go @@ -20,12 +20,12 @@ package state_daemonset import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_deployment/state_deployment.go b/metricbeat/module/kubernetes/state_deployment/state_deployment.go index 02448afdc9a..dde5f25525d 100644 --- a/metricbeat/module/kubernetes/state_deployment/state_deployment.go +++ b/metricbeat/module/kubernetes/state_deployment/state_deployment.go @@ -20,12 +20,12 @@ package state_deployment import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_job/state_job.go b/metricbeat/module/kubernetes/state_job/state_job.go index 1b072eae742..282bfa670d0 100644 --- a/metricbeat/module/kubernetes/state_job/state_job.go +++ b/metricbeat/module/kubernetes/state_job/state_job.go @@ -20,12 +20,12 @@ package state_job import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_node/state_node.go b/metricbeat/module/kubernetes/state_node/state_node.go index 99d1f351367..c6bdd890e91 100644 --- a/metricbeat/module/kubernetes/state_node/state_node.go +++ b/metricbeat/module/kubernetes/state_node/state_node.go @@ -20,12 +20,12 @@ package state_node import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go index 774a6dcd94f..eee1ee54ed2 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go @@ -20,8 +20,8 @@ package state_persistentvolume import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" diff --git a/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go b/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go index 93aa3ccce3a..99d784879c0 100644 --- a/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go +++ b/metricbeat/module/kubernetes/state_persistentvolumeclaim/state_persistentvolumeclaim.go @@ -20,8 +20,8 @@ package state_persistentvolumeclaim import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" diff --git a/metricbeat/module/kubernetes/state_pod/state_pod.go b/metricbeat/module/kubernetes/state_pod/state_pod.go index ea742ab26bd..17ed2f5a3fc 100644 --- a/metricbeat/module/kubernetes/state_pod/state_pod.go +++ b/metricbeat/module/kubernetes/state_pod/state_pod.go @@ -20,12 +20,12 @@ package state_pod import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go b/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go index 7ee8e62864a..9f3e7bc38bc 100644 --- a/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go +++ b/metricbeat/module/kubernetes/state_replicaset/state_replicaset.go @@ -20,12 +20,12 @@ package state_replicaset import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/state_service/state_service.go b/metricbeat/module/kubernetes/state_service/state_service.go index 6794c36c44f..7f996613860 100644 --- a/metricbeat/module/kubernetes/state_service/state_service.go +++ b/metricbeat/module/kubernetes/state_service/state_service.go @@ -20,11 +20,11 @@ package state_service import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) func init() { diff --git a/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go b/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go index c2c02195206..f0eeab506d9 100644 --- a/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go +++ b/metricbeat/module/kubernetes/state_statefulset/state_statefulset.go @@ -20,12 +20,12 @@ package state_statefulset import ( "fmt" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) const ( diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index c6f341090cb..8afdbdbbd70 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -25,16 +25,15 @@ import ( k8sclient "k8s.io/client-go/kubernetes" - "github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata" - "github.com/elastic/elastic-agent-libs/mapstr" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) // Enricher takes Kubernetes events and enrich them with k8s metadata @@ -538,8 +537,8 @@ func GetClusterECSMeta(cfg *conf.C, client k8sclient.Interface, logger *logp.Log return nil, fmt.Errorf("fail to get kubernetes cluster metadata: %w", err) } ecsClusterMeta := mapstr.M{} - if clusterInfo.Url != "" { - ShouldPut(ecsClusterMeta, "orchestrator.cluster.url", clusterInfo.Url, logger) + if clusterInfo.URL != "" { + ShouldPut(ecsClusterMeta, "orchestrator.cluster.url", clusterInfo.URL, logger) } if clusterInfo.Name != "" { ShouldPut(ecsClusterMeta, "orchestrator.cluster.name", clusterInfo.Name, logger) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index eb62bb49152..f53e16d37e2 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -30,7 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "github.com/elastic/beats/v7/libbeat/common/kubernetes" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/volume/volume.go b/metricbeat/module/kubernetes/volume/volume.go index 1b8da935889..e110c43761d 100644 --- a/metricbeat/module/kubernetes/volume/volume.go +++ b/metricbeat/module/kubernetes/volume/volume.go @@ -20,14 +20,13 @@ package volume import ( "fmt" - conf "github.com/elastic/elastic-agent-libs/config" - - "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" + conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go index fcbf2d0048d..39e0e6941f9 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go @@ -16,8 +16,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" - helper "github.com/elastic/beats/v7/libbeat/common/docker" "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipereader" + helper "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/functionbeat/manager/core/makezip.go b/x-pack/functionbeat/manager/core/makezip.go index 26bd9bc9965..aa311db9c34 100644 --- a/x-pack/functionbeat/manager/core/makezip.go +++ b/x-pack/functionbeat/manager/core/makezip.go @@ -13,9 +13,9 @@ import ( "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/x-pack/functionbeat/config" "github.com/elastic/beats/v7/x-pack/functionbeat/manager/core/bundle" + "github.com/elastic/elastic-agent-libs/keystore" ) func rawYaml() ([]byte, error) { diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 918946d2256..cd67cf797dc 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -12,12 +12,12 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/keystore" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go index 59470bf19c3..a157d287882 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go @@ -13,10 +13,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/test" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index b6236b4ca71..f1fa1ee8646 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -12,12 +12,12 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/keystore" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go index 00fa93b08f4..9d050765781 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go @@ -14,9 +14,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/beats/v7/libbeat/common/bus" - "github.com/elastic/beats/v7/libbeat/keystore" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" + "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go index d9eb4c45fbf..8779c99b7e6 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common/bus" + "github.com/elastic/elastic-agent-autodiscover/bus" ) // TestEventAccumulator defined a list of events for testing diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go index 8a67c966697..eb9f72edc8f 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go @@ -14,11 +14,11 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/builder" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/keystore" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go index ce90a94bd9a..d3f55cf88b6 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go @@ -16,9 +16,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/beats/v7/libbeat/common/bus" "github.com/elastic/beats/v7/libbeat/tests/resources" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" + "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 0a21b9c0744..33c6e2f1396 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -5,8 +5,8 @@ package task_stats import ( - "github.com/elastic/beats/v7/libbeat/common" - helpers "github.com/elastic/beats/v7/libbeat/common/docker" + helpers "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/elastic-agent-autodiscover/utils" ) // container is a struct representation of a container @@ -40,7 +40,7 @@ func deDotLabels(labels map[string]string) map[string]string { for k, v := range labels { // This is necessary so that ES does not interpret '.' fields as new // nested JSON objects, and also makes this compatible with ES 2.x. - label := common.DeDot(k) + label := utils.DeDot(k) outputLabels[label] = v }