diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8aaefc5e7a52..ea3fcc74035a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update to ECS 1.7.0. {pull}22571[22571] - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867] - Remove id_field_data {pull}25239[25239] +- Removed beats central management {pull}25696[25696], {issue}23908[23908] - MacOSX minimum supported version set to 10.14 {issue}24193{24193} *Auditbeat* diff --git a/filebeat/docs/howto/load-ingest-pipelines.asciidoc b/filebeat/docs/howto/load-ingest-pipelines.asciidoc index db0e3f00fb38..68471845a3a3 100644 --- a/filebeat/docs/howto/load-ingest-pipelines.asciidoc +++ b/filebeat/docs/howto/load-ingest-pipelines.asciidoc @@ -3,10 +3,8 @@ The ingest pipelines used to parse log lines are set up automatically the first time you run {beatname_uc}, assuming the {es} output is enabled. If you're sending -events to {ls}, or plan to use -<>, you need to -load the ingest pipelines manually. To do this, run the `setup` command with -the `--pipelines` option specified. If you used the +events to {ls} you need to load the ingest pipelines manually. To do this, run the +`setup` command with the `--pipelines` option specified. If you used the <> command to enable modules in the `modules.d` directory, also specify the `--modules` flag. For example, the following command loads the ingest pipelines used by all filesets enabled in the system, nginx, @@ -44,4 +42,4 @@ PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules system,nginx,mysq TIP: If you're loading ingest pipelines manually because you want to send events to {ls}, also see -{logstash-ref}/filebeat-modules.html[Working with {beatname_uc} modules]. \ No newline at end of file +{logstash-ref}/filebeat-modules.html[Working with {beatname_uc} modules]. diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index 5193aef8667d..fc9eaefdc329 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -49,8 +49,6 @@ include::./configuring-howto.asciidoc[] include::{docdir}/howto/howto.asciidoc[] -include::{libbeat-dir}/shared-central-management.asciidoc[] - include::./modules.asciidoc[] include::./fields.asciidoc[] diff --git a/libbeat/docs/shared-central-management.asciidoc b/libbeat/docs/shared-central-management.asciidoc deleted file mode 100644 index 6789ec012eff..000000000000 --- a/libbeat/docs/shared-central-management.asciidoc +++ /dev/null @@ -1,256 +0,0 @@ -[[configuration-central-management]] -[role="xpack"] -= {beats} central management - -[partintro] --- - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -[WARNING] -======================================= -When you use central management, configurations are stored centrally in {es}. To -prevent an attacker from leveraging the configurations to attack your -infrastructure, you must secure {es} and {kib} before using central management. -See {ref}/secure-cluster.html[Secure a cluster]. -======================================= - -{beats} central management provides a way to define and manage configurations in -a central location in {kib} and quickly deploy configuration changes to all -{beats} running across your enterprise. - -To learn more, see <>. - -To use central management, <>, then use -the {cm-ui} UI in {kib} to create and apply the configurations. - -[NOTE] -===== -This feature requires an Elastic license that includes {beats} central -management. - -include::shared-license-statement.asciidoc[] -===== - --- - -[[how-central-managment-works]] -[role="xpack"] -== How central management works - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -{beats} central management uses a mechanism called configuration tags to group -related configurations. You define configuration tags in the {cm-ui} UI in {kib} -after <> - -A _configuration tag_ is a group of configuration blocks that you can apply to -one or more {beats}. A tag can have configuration blocks for different types of -{beats}. For example, you might have a tag called `development` that you use to -group all configurations that are valid for running {beats} in your development -environment. The `development` tag might have: - -* Two Metricbeat module configuration blocks: one that reads system metrics and -another that reads metrics from Apache HTTP servers -* Two Filebeat module configuration blocks: one that reads Apache HTTP server -logs and another that reads system logs -* One heartbeat monitor configuration block: checks that a public facing website -is live. -* One Elasticsearch output configuration block that sends the output to your -{es} development cluster - -[role="screenshot"] -image::./images/configuration-blocks.png[Screen showing configuration blocks in a tag] - -You apply the tag to any {beats} that will use the configurations defined in the -tag. - -[role="screenshot"] -image::./images/enrolled-beats.png[Screen showing {beats} with tags applied] - -When the enrolled {beats} run, each Beat uses the configuration blocks that are -valid for its type. - -You can add, modify, or remove configuration blocks from a tag. Any changes that -you make to the configuration blocks in a tag are automatically applied to all -{beats} that have that tag. - -You can add or remove tags to change the set of configuration blocks applied to -your {beats}. For example, after you've tested configurations in your -`development` environment, you can remove the `development` tag and add a -`production` tag that has an {es} output configuration block for sending the -data to your production cluster. - -[role="screenshot"] -image::./images/enrolled-beats-dev-prod.png[Screen showing {beats} with development and production tags applied] - -You can apply multiple tags to a Beat. For example, instead of defining the -apache modules under the `development` tag as described earlier, you could -create a separate tag called `apache` that contains the Apache module -configurations, then apply the tag to all {beats} running on Apache servers. -This would enable you to maintain your Apache module configurations under a -single tag, while also using a `development` tag to send output for some -instances to your development cluster. - -[role="screenshot"] -image::./images/enrolled-beats-apache.png[Screen showing {beats} with multiple tags] - -You can apply as many tags as you need. Just keep in mind that the -configurations for all assigned tags are merged, which means that you should not -specify conflicting configurations. If there are errors in the configuration, -you'll see an Error status in the {cm-ui} UI and need to look at the logs for -the Beat to troubleshoot the problem. - -[[enroll-beats]] -[role="xpack"] -== Enroll {beats} in central management - -include::{asciidoc-dir}/../../shared/discontinued.asciidoc[tag=cm-discontinued] - -You need to enroll {beats} to register them in -<> and establish -trust. Enrolled {beats} will have the credentials needed to retrieve -configurations from {kib}. - -During the enrollment process: - - . The Beat contacts {kib} and tries to register - . {kib} registers the Beat instance and returns an access token - for configuration polling - . The enroll command creates a backup of your configuration and then - **overwrites the current settings** so they can be managed centrally - -To enroll {beats}, use either <> -or <> enrollment. - - -[float] -=== Prerequisites - -* Verify that your Elastic license includes the {beats} central management -feature. -+ -include::shared-license-statement.asciidoc[] - -* {kibana-ref}/using-kibana-with-security.html[Enable security] in {kib} to -ensure that only users with sufficient privileges are able to access {beats} -configurations. -* Assign the `beats_admin` role to any users who need to enroll {beats} or -manage configuration settings in central management. -ifndef::no_dashboards[] -* If you plan to use the sample {kib} dashboards provided with {beatname_uc}, -<> before enrolling the -Beat. -endif::[] -ifeval::["{beatname_lc}"=="filebeat"] -* If you plan to define module configurations in central management, set up the -ingest pipelines before enrolling the Beat. For more information, see -<>. -endif::[] - -[float] -[[token-based-enrollment]] -=== Token-based enrollment - -Token-based enrollment is recommended if you are enrolling {beats} manually. - -To use token-based enrollment, go to {kib} -> Management -> {beats} and click -`Enroll Beat`. Select the Beat type and operating system, then copy and run the -command for enrolling the Beat. - -The command has this format: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll KIBANA_URL TOKEN ----------------------------------------------------------------------- - -*`KIBANA_URL`*:: -The URL of the {kib} instance you will use for central management. - -*`TOKEN`*:: -The enrollment token generated by the {cm-ui} UI. The enrollment token will -expire as soon as it's used. - -For example: -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll http://xyz.gov:5601 70f4b584e8024b96b682c46125a8d81 ----------------------------------------------------------------------- - -Repeat this process to enroll additional {beats}. - -// Maintainers: If you update the following note, also update the note that -// appears later in this file. - -[IMPORTANT] -===== -*Windows users:* If you installed {beatname_uc} as a service, you must also set -`-path.data` to +"C:{backslash}ProgramData{backslash}{beatname_lc}"+ when you -run the enroll command. For example: - -+.{backslash}{beatname_lc}.exe enroll http://xyz.gov:5601 70f4b584e8024b96b682c46125a8d81a --path.data "C:{backslash}ProgramData{backslash}{beatname_lc}"+ - -Why? The service installation script, +install-service-{beatname_lc}.ps1+, -changes the default data path to match the convention used for Windows. If you -run the enroll command without specifying the correct data path, {beatname_uc} -will be enrolled in central management with the wrong UUID and unable to receive -the configuration. -===== - -[float] -[[username-password-enrollment]] -=== Username and password-based enrollment - -You can also enroll by specifying a username and password. This is the -recommended way for scripted deploys: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll KIBANA_URL --username USER --password METHOD [--force] ----------------------------------------------------------------------- - -*`--username USER`*:: -The username to use for password-based enrollment. The default -username is `elastic`. - -*`--password METHOD`*:: -The method to use for getting the password. Available options are: - - * `env:VAR_NAME` gets the password from the environment variable `VAR_NAME` - * `stdin` prompts the user for a password. This is the default. - -*`--force`*:: -Overwrites the current settings without asking for confirmation. - -For example: - -["source","shell",subs="attributes"] ----------------------------------------------------------------------- -{beatname_lc} enroll http://xyz.gov:5601 --username myuser --password stdin ----------------------------------------------------------------------- - -[IMPORTANT] -===== -*Windows users:* If you installed {beatname_uc} as a service, you must also set -`-path.data` to +"C:{backslash}ProgramData{backslash}{beatname_lc}"+ when you -run the enroll command. For example: - -+.{backslash}{beatname_lc}.exe enroll http://xyz.gov:5601 --username myuser --password stdin --path.data "C:{backslash}ProgramData{backslash}{beatname_lc}"+ - -Why? The service installation script, +install-service-{beatname_lc}.ps1+, -changes the default data path to match the convention used for Windows. If you -run the enroll command without specifying the correct data path, {beatname_uc} -will be enrolled in central management with the wrong UUID and unable to receive -the configuration. -===== - - - -//[[central-management-API]] -//== Enrollment API (not documented for beta) -// -//available. - diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 42a00fe97af5..8fff64854e82 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -49,8 +49,6 @@ include::./configuring-howto.asciidoc[] include::{docdir}/howto/howto.asciidoc[] -include::{libbeat-dir}/shared-central-management.asciidoc[] - include::./modules.asciidoc[] include::./fields.asciidoc[] diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index 7e805a92e7dd..213a0c3c68a8 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -7,10 +7,10 @@ package cmd import ( auditbeatcmd "github.com/elastic/beats/v7/auditbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of the beat @@ -23,5 +23,4 @@ func init() { settings := auditbeatcmd.AuditbeatSettings() settings.ElasticLicensed = true RootCmd = auditbeatcmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, auditbeatcmd.Name) } diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 93bc6e722cbe..fee17842e8d0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -25,7 +25,7 @@ func init() { // spec/metricbeat.yml // spec/osquerybeat.yml // spec/packetbeat.yml - unpacked := packer.MustUnpack("eJzUWlt3qziTfZ+f0a9z+bjE6WbW6gfbaWSwQ45xLAm9IWEDtoTdMb7ArPnvs8TNgO2cW8/0fA9nnYQIqVSq2rVri//65bBfsX/4e/Hvh9XHafXxH5ngv/znL1SYKXnfhXM4cmbQ4SwhnIX7DUXzZwuYZ7pQc4JtjWBr6mFb8RGJPP3u3xKW70J03oXW2ErdhXWwxnbqoUFENJgSNFBmAh49ZB8ImhvBxFbJwjqM42FoxappxefQEmpEhZkQpHKazI8UGVvyrj7N8CjytENKxIUTPI9nw84aEQNKyjSYBQJmM+FGATDXTHczgsyUZQNOhRlTALdfsJsFaNl/P/FU40ywvZE2flm4v0pfLBW4gFu4wErwxVVen1/fh+f1wpri+T4L0IWzfDe1xsPUAs6JTfg6AMaaAp4HL8XzUP7zscvp+y6s7G6eB+C3ZwvwI5tAhRX+ks9HJ8qNDdUGggK+CV52IRXGiRW2KiFF5vktbMaePezu3uKRoADyYNg8b9tWPRum1sTmFBkagcYHwdvnwufj1j+Q8tX7LvSQsyHYybFmnv1y7A/NwwQ8U13GyiBpj5ktRhlB6ikQcO1rcHDdT/1vVMxrARkDA0W+O1uMhI+j5jzf4tGv63npEyL4IQAwwzqJ6AR29s00eCDIUahu3fN1uc7EHTCwvJ6LMA8BgnnXLiX08ChtbEEXhWDruveJwymAmwAY2Y3fb9Yt56M6VOTznm+qs9zrPoDHt3h0IGiQBCDc2ZN0T5ORGkwcY7oY/qv1Mgw9NNhaIIqYkvLVItyuNEMNJnKMcrDGAafAzAPAN0yDERPOzs6201/+rQSBVRLsd3GS9iDARYMtA8aeJvNwqcFNgO19MNlOPU3dvsUjToV7pho/BmM1J8hRmeDKar6PWOLuiTCLkCXXOVICoDZOCkjZe9ry2Xrx9LeXcOohR/GRccRa6R6sy2Mo3J5aAB7JZHTy5fGLy4moRhHqpWtHWw/buo+enq2xdXoHPGbCzFYLw6y3O1Ou7890R/Gwy2fa5UQyo2W/8udMzp1Zcs6Djwbq6mUXWrFxYpP5yUWXiOnu3ssM8/qOkQfAVMjCOFCNndr7nMYD+SyWxxlo/EiAoUsotLavz9i8zJkwEibM1PqD7CmAOTYvjb3Fz/Ua5oUxXcIXZBjIvV/Y3XWEsyPI+Sj8p7sRBefncayEBEfcUw0ZorwOYQZMxZd7Ey2/YId7Osx87A6salwF39M6RC1ZBgQXq4V1fRYrKQVGUr8zWwxjprtbgu2sfhYAnhJkqDIWXvPhlAEjD0xpv6N46HKozviJIGct05e8V2tORlEAwmdrbN+Ps9oOYGZEf63TL7XGdjN3267ZQm3OpBqXB8DlLLFaz6x0huGZ6HZEwLL33OZMM1QmHM6ylg8e+LE7fvDs42E130jxZTnTofIWD7XXl+GUTWyOdXj00UDG1IG+7KazxYivANxgTcbIstpfDfPDuB0H7Jqb9RoRE0EbtuR+VSqa+IivUHN7jvf9c8fuBB4Ifi1jTfAnrDsKEzCSJW6lK9Pq+X1IbsHkWzyq4Ox12i4bwYSfyXx3F4qlf5q4GBb+qqDYXvfHfgb9RdkpsKykBNVeNQ9d1HJvdakrqEKxp3bprfOpU9IntkpBx9bHZa4ukRVdGRf5QPZ04nK2ufVVOye7NEIJfTQ4B9jNG5tBVQKGLTsw2TONn2i4mwZaxOlmF1KJsbq7m47dX8s53V5ZuXAqAsUfy7JS+U9X9tbLU/g6HkVUzEMfmPlCgwM5R13O1otzaGvw4GGJ705OkJl5WphM53tJa/JgYkcybyQ22pM0C9CgiLGZkOMiwxr/YVjjIGJC0Zwxa8rVOuYruvJvypWED2RzD8/rElVAnydgFAz3ZUrEI9phmInDgwk8zwQ/0EWLGSIZrg634rN0bzxbLuPZeBgzDSoBHh4DAFMGLlEAlkeCBpEn3faiCg9d8r+IxX4nU76ciBbsqWBHCqBG0NkgAMYBYl9lt+6Wv7pL25xDd+0qxmSe7y6vLyPDCvfBWJhZALiQJfotHsUEmQrLjLZfM4JlWbUVrMkSXoWcaWx8YB6JLPNV6nwNLmaL0Z6KPfd0d+2jwZbgsIY9ZYVHXDIgqgV5mSL86Au4CUxDhhH3kLpmE/vkaTBnmpHVKUa1wdrTjCMRl33ZpfBjkWqmEZHE5awuUxJq3mUqGAk7F3CSenjYg90eTN2m+YbqowHWzAM1DYWqxsHHjtJJ94l7eotHtc15m/Hd2jo4UX3YYpCyJDicTebPlnmIrx1Nca57Mi7+b2K4jO9RRjWHM905seQ1xvOerbp7wtplz/R5h73XnUrbpx3W+l37aHweE0ROTJQwyoqYVhtqQhL7JNfswthIofn1WbPnc1UiJ24mY7Pcg7su7G5gu39uTs/e0V7m8e0+umuW8aZG7KVHTXql5Brf15LhoUFOJU18b3cCjV1FXHd8h8wzAzDrd0IyLit6VPzc7bRGPBBQUtKS6hV5wqbdTkxiQkmBCeDHYs/A3dPeOgQTiVGnADhnaZ/Xib/2PHDLXnZhgNxzz5aSDgNj42tQlvYt1ZwPgq3ePAUFzwhy90w1cgoMXe7rLR6Vz863+5/pzoBJ6p7vQqbzvOWHTzvRqsSrTIPrXqf1Le9tPexGDVbNv72kE2AqHqzxSTkWdQo7Zw85/IfXXwyK32Ud6lAFbvQoyygnEqN198Q2Nzl12+1O+h2z7HKDfIacA9XhNtBMxdPC7t/wa4tqmM8BtvkMlfHhT+CVPoNyDiIMlQo3W7UxSBsoHuLHa3woYR071/eDDRur5wDZeYCu4/yJqzDTaNPAet5dMHHPfuKc6BWbPzxEPrzr2q0cUUKKTaX1/pFgV8iYvNrwW+5sYBQgd9+hU61YnS26OSV/X2Gns47Mq+vZwy27ziXp+dP1b7KWX07kOneR2zX+/fTZg4ZPPDz/gmfMd5267KGA06Soz0JiZE3zPc04r2qMAySjmnKDj6XKVbS8BY/4Pqp75XYPVJO0yDssab7Nx0mwI+jpuUtdr2vPxM/T2Nl4mDABtz5+TUqOFZQxtmAHaxwUXCUAZu6P2X4c/v57Q1/5apXeF13dkgqGy7obEE5Krt1FI2pKWia7XkkDZgtV0kTZHadUc7nFlZ4wSw4EwaI7rcTcnmBJ9quGRinP1iQ1xuH/raB2s7+/av1+mH9iQx3un8JyBe1N6FV21rZgCUPgt0diYIiyStQbD2qh/FjPNRM3IRbWdK1FGU4Bds8Bnk8rAbJOdZmSkrbc8Y9dtD1NHCRVHMSDM9UkRdsefTS/t1ZNd4+v42Zsve6eFvO4awKg8DA8BJPXbxRBb+zYUd25L362/MSAkV2F7U7cHBuxP3FyOvx0H80FRbWPFs1vU7vSxuvzLtVvw1Jwh651IDh5vRXGyz2WeTvsv1uVN2DkwbDO0Rb9bKkOvY7/oZ1MwJTqhGOtiN9pf71PKNLX6U4n37qXLX/FHDMxiCiCOQPmhsx/aF+PKdSP2Dep5sGv0x+7BOjF1rjOp1Zr2YxtFJy/Ua3ZNaUrWvkf6R3pZQFgxBK3LOtVzfI7z1r1qiel+OiStuUNIswD08ox3yu7PJZP1BMBS2M1ViX9/pih8gyKFvmBTNOaX2EJ5N2xj+UW3LsBKWSSyeup44/EllRO/r6W7XJ9Blhzdh4aJKRQP2U9+oZa3YnBq/qJ9WAfgGjNBEwIjtqXhXdrMxPLZ8sc5CtJ3wEsYmCWPX3ci+meFPIJFf2K0vtQgb1HSWtqXMpJnXU3D+peDwdv8LLMvaamdnPva3Ty76OQYpV+xOxOIr4jqDDBN5U2V93aqzyY2HtPq7TR+zfzOcGuysaDPQXK1zXNWjdNXE7x6FBcq91L4k81zYdJ2dFcKTKPRd+BJMn96u29dFrio0EyExceCHj4glzuJTDpz0s0Z80AzDxZhF+U7UpRTWja9lz5Li00JdjNfOR8b2KfJenpXtmVhKQsaO1rn851mCT5RZGfifmJ6TyXoDhLeErHVuuLAjMjWnFVE1NkbAsQyQbbQkPRXU4XgwFF5yMBPK/OLB4nDXl4ADRFAv1ZE10C4NNbXz+t4q/S55q+sS7mWG/pkXXSAuOINedEBTn4pW8bcukhojTaQAlY13jWX/8qHfETrfdGM+x9UVH1vtCIKLjc/eqjXru1Zksrutn7kWpGR1MjONoQPFIKEp805KOIYR/Ny1iu87XUgM9MGMWXGxIEZUPYs/W+Ng24Qoqz7mjGP7qP6xkKKAoQra56g4kdyZgo/GQahQZ7vRbsaav6lWx1bew9L/XoprjcvXpUDd3H7g7rNicafGoI9Z3rxrIYPh2n1/zO7eGn14l/+xVkL95v4lT6nSZOgdWPyUHrDHoEQc4pcxzH4T+WL5dCu/4SP31MF7c+KueRa0jMcts6d9lcl/WqPbdgwkjvaeKlbqSeyAQeyFW/Sz2UcqyZGRPm4G4cNzjh8E6TVcZKYzNpX5l+XY9tvXfVoh43Io+EiG95524z9XOa8Y/PUWAs00eRpy2/fQ8TmBNotESLdg0pv+oqaon4rRaeqi/oapuauOjdvXXO8Kon9mOr0SI7HOh4rzY0dbKTy9atppq0crLHrWq/fKpNhne/6OrspeBGMt7xz3zhVfCdK8EcDxNfM4Wv/VH8XNx1Sq7Sa/J2hz+Pq4/sHrvUnUuAYLbq3rKfmG6qBNuD/k37d9yyfz+zfMwQCxn0Cw5TBsyNn6mVlPRZe3h7u/5NjJIrUyagpOhZYBonyutId9eeFkVUBFwimmwfH7DHG995uruXzLGK0HVRlapbc1r7f777Z7tN/5nK2KoId6oiMHKGIWfJ9hvR5dtuHD75mOauFFOPXy+24Zd4eLaAeSTj0c7DzozgrczwOjYMmX2k/qhQd7iH7U3R4gnnRAuGZ+5p4uZv8XC70m2VJu6eouWxGDdRQutdCW3NzOi7p9hZ2XY+/oCG5W8vXtJuH/c+267u6ThLmTAaVDrt40RCfsoD0GsfM5a6ZZ/6ldZRjrkZqxCknour8TuAYGfn0M5Us/hf+6bPZwpImy2XJbR93jJ2xwIjITJIssGhuP5+UbcE2SrJgkctX9GuEvzzWs5PaiZdqvRQL6nOr9GSm2Qv9WfT2FNBTtWH1P8rusr/D/1k9/sv//0v/xMAAP//PEBz+A==") + unpacked := packer.MustUnpack("eJzMWkmTozibvs/P6Os3C0s5u5mI72DIZrNNlnEZCd2QZAO2wO40XmBi/vuE2MF2LVnVM3Po6EpZaH2X53le/ddvp+OG/EdwTP7ttHm/bN7/PU/Yb//5G070DH05hEtPdeaew0iKGAmPOwyWL5ahX/FKLBC0JQStmQ9tIQAo8uWHv6WkOITgeggtzcrclXWyNDvzwSRCkpchMBHmiXf2gX1CYKlQ0xbRyjpp8TS0YlG34mtoJYMxz8jQBd9TCmrazAdi8e3v6Q7KKiOJw3C6VGwzU9d/il9czwauZ29dQTGXxeG2eFUVKzxSLbVFbHjFW8jXLIS+pFw3niJgUTkF0BGq9mloaeqRGl72FqsJNjxGp227gItDGIDJlUK3KNfC2w3lDCXnghN0CoAjvMXqGUvKlf8+X6mZD6cvbV9TjagRvlgGOiHgCV17f211myaEJPEyLCMGpYxtvhxm3W/Vf4HkTd5iNfIlhxHZ2fpQPZZ9lx8aJ0dQvZDUPeKE9PtklmkzDBQJeco7gvtuP81/Rjlu6APK72JWfmOg40Zv7lN4scxMqc8kCcBNQNDe0kQ/UdDft1ogcGO+7F7I7tFZV/NQk11Ru0dV8sFNRHAxWNd8pUbEENq1YNNlZNftnUjeCQFHwLJ9d+5381bjXSh0rxQuh2fT3GVKDwh8erGMG8MJFQIt3G8kdiamJxBZOFqvn8KFpkY4WYaBoRcryZvMNPd3LHsC77NdXUNb8k4+dIQAOAUCeu5LYTpbHv75279WDr1J6fEQp9nInV0w2RNDOeJ0Ga4lb0ehfaTmfuZL4v4tVhlO3CuW2JlqYoGAI5KECZvlMeJXjRJ9R18PIerGyJDhSVpahoejL61frFdffnsNZz7gS+PmXm0Lyu6EVGabWYZ3RqZ6CcBE0JLbBYnK1YfugRT8yNW9D2054MejWZcvBotJoueblaJjQy+owXZzoft+LjuCD102l24XlCu99Qt/zfnYucXHPAVgIm5eD6EVKxdiLi8uuEVEdo9+rujdN0pBDV1AK+WEJXLp73MWT3hbzK+ZSuyMDEXmYc3aL16gfluSRElJomfWn+jI3RPqt3a95b+bOfQbIbIbUcMj0OB7v5GH8yTOAQHnvTw/2Y2wcX3RYiFEMGK+qHC3YI0pEkMXAr63pHcu0GG+7OUBdCdW3a8OxbPGdC0e0hOWbFZW1xYLGTaUtPlmvprGRHb3CNp500YNliGgiNwWFsV0RgyloDpfvyP44Haq7/gTAs4WJeyEmjDShDTNfmxnzToMPUdy66KZpdnt2P11zVdieyd1v4IaLiOp1Wuzsjn0rki2I2SsR+02I5Ii8rRA8t4ZPDnHYf/JSwCn9XiqEACRcfd8i6fS4nU6I6bNoOydAzDhNnXCr4fZfKWyjeHtoMRtZF3vTy1t/y2exn07IJ1vNnNEJKH98MP3K+KktY+4C0H39/j4fB6sO/VOVXgUQpSwT1B2BJJ4Ef5yCDeyMKvbT9TwciijCJsea23KdBg2vB01lPwtVo84VUVqLmbPwvE4pPPzae1iWp7X0/DfhWTrxTLq0Hk99FMUw4keY8Pb13sdh/7MMt2cgnW5Jwz069ifUHJjqAnho7Q7X6k5AuKFJt62nK+fFuq9EsnLaeLlWukPo7TSO6u+Tw7mvE8vmWUoIjVVsYEa5TogOhKJXXB4mFEpYnh3CDGPsbJ74GmjGtNVZqvpP6zXaeiDyf7Xp57jDksTnsIj7jc8NtpmllMwKW1snvB+kWJpfyqWRiOSCJKjkVmTrrYx2+BNcJeuePgANvPhsklRZejzEy+i02PlErGKB2gvdRg1ves8YSe8mrRm8Blwc3WYFV/58cbz9Tqea9OYSJ5A4fTMkRwxbhE11mcEJpHPj+1VTHxwK+4RpRjhRE8Rd5902e8vkNS7m4O7IuJpI5+cEEQMv4p7BGwR5d9EqsZqfdOXe0/1dMX8ItDXt92f14UpxBoTRsibn5NbzMvw4sUI6IKW2oyHB5K6WyxNts01Qsk5+GCSotIlbREtjzkFt9KVS7eD0ZbIbo6AztHt79slN2F2DhJvV4b6xGWb18aUeWpfczRzQfKidKcATP7i7tmGD0+5kkTZIegU3GVrl7xgpnCzSbDBSmjBwyWCtgAlPeEhpglTG6iyt1g9YYkWlTuJEXkdpZVRGOhCWeM6zoWYbMvTCJTdC5RuRyIvB0iQGn+8WGa9Zrjoobf7teJEuZBp3+29T77kXflvILdbdlLdK9tX/2+ZSmV7pn3xJa8gkpKT3KbjtVJD2WKDFfS1j8Rb1tE7U7v46D66M7cZSpQcLUsbyLlNY9CmqYQkSnYX1k330rU57Z61GgLwcODLbrUHXSnX3aWL0b3Jo/UaTEAlGxi1D+ZcPA3fw9CptvbdhFa+Npw6Jw4fB+G7WVdl1/2zy3yoXhG0RqymtMsKkpU2SoasyfCkkuXWab/0k+thwIrKmJAuLxxqldDXdARksPNoHkYTj8NUwZenfH27gf31xqHAvb7FqojM6WgtJUzeY8l55/uwDPfiSxkj4XAcHq/mss2QwQooOycsU76vF87ueNv9/smFyKzg373FarGBTu8cvsYq1ZIlWqZXIE8Zsabv+M7gUFpvY1WX2p+z0Boe7H3oRm18Wk3OPhAZkTlDXn94/nlS/l3wFP43Q6WIyovMl278rmUfurtgOvyNFIt2Hz48iiRZZ5V9uAcKOrhbj5FgmcNee9KPQTh1eTpv7WO+Uhvb6SCL5FznUBX91BH9rt+Bmu4VSj2q144bCdRU/yKScu7asgglWdT93fnIfKVmBLq97yeMGuiE5c6+cLGQHKCLyGBC3wZ6tpqNfIr/PSHSYB7uV118AO616+udAxh2v0nszG29W1NF+2oF46fvvsUT06f3X+KMKs62eTlCkneu8jO61Hmbx/IdltUWoqLUvnA4P4qPpWKFenirv4d72G3319LDZIcnCohQ+h2R3QtJ1k9gajv3uVnbdrUPP8fTq2XoZ6SpBx86cwT3B9vM6vFdZa5NU9TQedlhPrR3gUZOlkZzBNwjyUmJsWypsjE75xCWYxVH8KFzsPN9B0PZZpM9FkLdCpmH6wbVJ06GOpaQtWg/qdirpZ84Q62uRBMzLLnMuoNslSDEWWYtsPbT1GNhrBYln5rXN9hRF3KH6e1eYBswkqzHZn7N/EYLhb65hgqi1mfyLLTXrtNA3GadzVogZ5XGHz2BdcDAQpDXIq42aYXmZqx5csd4QtiJijWD70y+Fi8bgXLLwx9+eD52SV9aO0hrO4gnVyzdjr68Pwdg+WiuJmycF1rbt5n3iMtx3C0yvMSH3omai2f7fihQ9tZxwLIjjMTNu3MihpLT6UO7OTd2M0+dAk+/uo+2aPArBPFx6Oql7k44Thf3AnWdQku/nY6/rUOZoRR02vhoj4o8hZ8/LdyLRPJa+ja6j+dKxLeh0k+NMU84RfAKYug79LFCwhiGlX9zSv6h9fWowrCAYP2Q/Y/mHqbotm+rxPwfqi6d4B9tgvfsgYSyMjxO/ys5oM5ZwaCtl69GkkgAblm/uIYS/USkqs+Pyic/Uvzr9eXUKg3AJJ0nN05/Tp+By/zUS+9zaSN/RBwCCLVElCPI0zynt8oZSvWd6cou4JBCWr80KmBfLngigTzOe6IiB9A9QE6XJO9TPxbUNl7KF704wGnihsg8H0WMGmxH8j/Os+u3C1u/oiA2hpZfKYpVEPNDfjTO+TVMfxy/3hFkBR8HLY9yYHhnTqMRmKTUCDnUa+SFkS9FEREytllxX6rv1RQ47GNN5YVIXkQSDvGuoV3TWTsvpbk0kPQkkP5MOXzkUL6UyUY+lWyy95g8cKovwBNIwna1kdVVcZFR0z76Uq1XPq58Fwi6ItEmR2wI39YZGy0zdRmGaqlVPHTI6c9U128XJNEjTsgZl1rFVUGGF1NAxuOmvqhcEbR3fNzPK/f3L2tvvd6z1x/TJ4fnRBKvoIaeU125YNZoAO7Wl6IIJ5Q7S2WIqXohFZhqSyp1Ai4Tw6DUNCiLiRdkls59RppSbKAjICCcN0A8deURlQeJFMHlCz8/LLllsp4ny1LX4MFtnrIMa5N9AJ3qzjTrm+WSMnkkxwYE7hEMX0aaZoagm/OgPuJyjZa27WuEjbNjabL1JeWMEg78lhXI5kCLc8DUbfl6qSFwUFzbK7n+Km3vK/rrWMe7L52UfBRK+gnrj15VtHN3c/YCxv3eJxcs93UulW0MhxFzWSa1JoiTvLThI9LK/7d1gcrP1BxXLyM4KIwbwN2u9YleHECX8bse6LjXj+6jvcMYAVTy5LrkK5BE5zZRnhOUmICAWDzTO9v9NuWwZo2j9lIj7gL1w3IblOmRGtGWJF6KYNRqrPeJTc25ncH403vr3/KiKb09Tkz9shhTRknoa8nsCYE0x0S0I45DMnFHPp7ZKT/3M5E4qPG2VGJCoCs5ApRtzOnADpo7GL426ZL97FVZfq4I9z/m8el4f0Y1AeZzvB5Ce0DOSzBd6pPDMiPKsSQ80qlLLQcDZU/BjXUalRgFkrf1oZ37Y020tpE2TowAeGUrzZp5Tlr8gEba+276/eXTMVD5rm8GL8OEX6LjfnyMe3L0PXug0GEtUK3K4G0OqV5Nlbkk4XfclOTLF2rNmhq7GIGwwR1KncY3tq1GH8R9jMPxyn1uaPPkj5S4B+N+p174+MXUYC/nxt5/hlBVeKfTDS2NvvsAvfur8t9l/ZHbRaCRoxb+swWKh9Nf5817/ggpys6NAi/fDKvYFyLrIoL2ZFzJ/oEq9o+jxH5FGujn0pqAd6Zab3xY0qFh36fVa5v+QIV58Iis3Le5uODx+Xz14ZhSEOgxku5nvTeaT94P9qrQ5YMt79PbGBVx2H8deljjvVDuIYLGou/fb7ZSmw+Q0FZFHlTE/8ZKUGtL31kNGFG5h3LYHYXre+jf937xQ49IKjqoub+T1DuXj9NWk5L2bVfT6PNqmi6KULKvHa07BmS/eaSVrA19F0ieMKB1phr5UsaoMaJ1OcncKix8g9LxPnd9OYS74vK14b1zl3w1F/WqVPH1pybDvk+pXAqfOCAZ7vnjesn/zkPdqw+cd9Rp7N9IjX+DdhH+v9AoZr/997/8TwAAAP//TywPhg==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/spec/apm-server.yml b/x-pack/elastic-agent/spec/apm-server.yml index 7e9d2af76dc4..67aac6349e39 100644 --- a/x-pack/elastic-agent/spec/apm-server.yml +++ b/x-pack/elastic-agent/spec/apm-server.yml @@ -3,7 +3,6 @@ cmd: apm-server artifact: apm-server args: [ "-E", "management.enabled=true", - "-E", "management.mode=x-pack-fleet", "-E", "apm-server.data_streams.enabled=true", "-E", "gc_percent=${APMSERVER_GOGC:100}" ] diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 0bf33f15e7d0..8207c803284f 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -3,7 +3,6 @@ cmd: filebeat args: [ "-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", - "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug", "-E", "gc_percent=${FILEBEAT_GOGC:100}" diff --git a/x-pack/elastic-agent/spec/heartbeat.yml b/x-pack/elastic-agent/spec/heartbeat.yml index b4f5b14e5a87..adb0c414f106 100644 --- a/x-pack/elastic-agent/spec/heartbeat.yml +++ b/x-pack/elastic-agent/spec/heartbeat.yml @@ -1,6 +1,6 @@ name: Heartbeat cmd: heartbeat -args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.enabled=true", "-E", "logging.level=debug"] artifact: beats/heartbeat restart_on_output_change: true rules: diff --git a/x-pack/elastic-agent/spec/metricbeat.yml b/x-pack/elastic-agent/spec/metricbeat.yml index cb4c1e8ca2b9..7968de13e8d9 100644 --- a/x-pack/elastic-agent/spec/metricbeat.yml +++ b/x-pack/elastic-agent/spec/metricbeat.yml @@ -3,7 +3,6 @@ cmd: metricbeat args: [ "-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", - "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug", "-E", "gc_percent=${METRICBEAT_GOGC:100}" diff --git a/x-pack/elastic-agent/spec/osquerybeat.yml b/x-pack/elastic-agent/spec/osquerybeat.yml index bb6e7e50a891..4735180343de 100644 --- a/x-pack/elastic-agent/spec/osquerybeat.yml +++ b/x-pack/elastic-agent/spec/osquerybeat.yml @@ -1,6 +1,6 @@ name: Osquerybeat cmd: osquerybeat -args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true", "-E", "logging.level=debug"] +args: ["-E", "setup.ilm.enabled=false", "-E", "setup.template.enabled=false", "-E", "management.enabled=true", "-E", "logging.level=debug"] restart_on_output_change: true artifact: beats/osquerybeat action_input_types: diff --git a/x-pack/elastic-agent/spec/packetbeat.yml b/x-pack/elastic-agent/spec/packetbeat.yml index bf05f901c48f..37c2629f1300 100644 --- a/x-pack/elastic-agent/spec/packetbeat.yml +++ b/x-pack/elastic-agent/spec/packetbeat.yml @@ -1,6 +1,6 @@ name: Packetbeat cmd: packetbeat -args: ['-E', 'setup.ilm.enabled=false', '-E', 'setup.template.enabled=false', '-E', 'management.mode=x-pack-fleet', '-E', 'management.enabled=true', '-E', 'logging.level=debug'] +args: ['-E', 'setup.ilm.enabled=false', '-E', 'setup.template.enabled=false', '-E', 'management.enabled=true', '-E', 'logging.level=debug'] artifact: beats/packetbeat restart_on_output_change: true rules: diff --git a/x-pack/filebeat/cmd/root.go b/x-pack/filebeat/cmd/root.go index b95a3cb9e969..18bdc321b30b 100644 --- a/x-pack/filebeat/cmd/root.go +++ b/x-pack/filebeat/cmd/root.go @@ -7,11 +7,11 @@ package cmd import ( fbcmd "github.com/elastic/beats/v7/filebeat/cmd" cmd "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register the includes. _ "github.com/elastic/beats/v7/x-pack/filebeat/include" inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) const Name = fbcmd.Name @@ -21,6 +21,5 @@ func Filebeat() *cmd.BeatsRootCmd { settings := fbcmd.FilebeatSettings() settings.ElasticLicensed = true command := fbcmd.Filebeat(inputs.Init, settings) - xpackcmd.AddXPack(command, Name) return command } diff --git a/x-pack/heartbeat/cmd/root.go b/x-pack/heartbeat/cmd/root.go index 8e25993bd877..d777f631e485 100644 --- a/x-pack/heartbeat/cmd/root.go +++ b/x-pack/heartbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( heartbeatCmd "github.com/elastic/beats/v7/heartbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to handle beats cli @@ -17,5 +18,4 @@ func init() { settings := heartbeatCmd.HeartbeatSettings() settings.ElasticLicensed = true RootCmd = heartbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, heartbeatCmd.Name) } diff --git a/x-pack/journalbeat/cmd/root.go b/x-pack/journalbeat/cmd/root.go index 7d27c7e8feff..d549011f73a3 100644 --- a/x-pack/journalbeat/cmd/root.go +++ b/x-pack/journalbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( journalbeatCmd "github.com/elastic/beats/v7/journalbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to handle beats cli @@ -17,5 +18,4 @@ func init() { settings := journalbeatCmd.JournalbeatSettings() settings.ElasticLicensed = true RootCmd = journalbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, journalbeatCmd.Name) } diff --git a/x-pack/libbeat/cmd/enroll.go b/x-pack/libbeat/cmd/enroll.go deleted file mode 100644 index 3f44647bd5bc..000000000000 --- a/x-pack/libbeat/cmd/enroll.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package cmd - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cli" - "github.com/elastic/beats/v7/x-pack/libbeat/management" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -func getBeat(name, version string) (*instance.Beat, error) { - b, err := instance.NewInitializedBeat(instance.Settings{Name: name, Version: version}) - if err != nil { - return nil, fmt.Errorf("error creating beat: %s", err) - } - return b, nil -} - -func genEnrollCmd(name, version string) *cobra.Command { - var username, password string - var force bool - - enrollCmd := cobra.Command{ - Use: "enroll []", - Short: "Enroll in Kibana for Central Management", - Long: `This will enroll in Kibana Beats Central Management. If you pass an enrollment token - it will be used. You can also enroll using a username and password combination.`, - Args: cobra.RangeArgs(1, 2), - Run: cli.RunWith( - func(cmd *cobra.Command, args []string) error { - beat, err := getBeat(name, version) - if err != nil { - return err - } - - kibanaURL := args[0] - - if username == "" && len(args) == 1 { - return errors.New("You should pass either an enrollment token or use --username flag") - } - - // Retrieve any available configuration avaible for Kibana, either - // from the configuration file or using `-E`. - kibanaRaw, err := kibanaConfig(beat.Config.Management) - if err != nil { - return err - } - - // retrieve an enrollment token using username/password - config, err := api.ConfigFromURL(kibanaURL, kibanaRaw) - if err != nil { - return err - } - - confirm, err := confirmConfigOverwrite(force) - if err != nil { - return err - } - - if !confirm { - fmt.Println("Enrollment was canceled by the user") - return nil - } - - var enrollmentToken string - if len(args) == 2 { - // use given enrollment token - enrollmentToken = args[1] - } else { - // pass username/password - config.IgnoreVersion = true - config.Username = username - config.Password, err = cli.ReadPassword(password) - if err != nil { - return err - } - - client, err := api.NewClient(config) - if err != nil { - return err - } - enrollmentToken, err = client.CreateEnrollmentToken() - if err != nil { - return errors.Wrap(err, "Error creating a new enrollment token") - } - } - - err = management.Enroll(beat, config, enrollmentToken) - if err != nil { - return errors.Wrap(err, "Error while enrolling") - } - - fmt.Println("Enrolled and ready to retrieve settings from Kibana") - return nil - }), - } - - enrollCmd.Flags().StringVar(&username, "username", "elastic", "Username to use when enrolling without token") - enrollCmd.Flags().StringVar(&password, "password", "stdin", "Method to read the password to use when enrolling without token (stdin or env:VAR_NAME)") - enrollCmd.Flags().BoolVar(&force, "force", false, "Force overwrite of current configuraiton, do not prompt for confirmation") - - return &enrollCmd -} - -func kibanaConfig(config *common.Config) (*common.Config, error) { - if config != nil && config.HasField("kibana") { - sub, err := config.Child("kibana", -1) - if err != nil { - return nil, err - } - return sub, nil - } - return common.NewConfig(), nil -} - -func confirmConfigOverwrite(force bool) (bool, error) { - if force { - return true, nil - } - - return cli.Confirm("This will replace your current settings. Do you want to continue?", true) -} diff --git a/x-pack/libbeat/cmd/inject.go b/x-pack/libbeat/include/include.go similarity index 69% rename from x-pack/libbeat/cmd/inject.go rename to x-pack/libbeat/include/include.go index 6879e0358e4e..bf61ff599d92 100644 --- a/x-pack/libbeat/cmd/inject.go +++ b/x-pack/libbeat/include/include.go @@ -2,16 +2,12 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package cmd +package include import ( - "github.com/elastic/beats/v7/libbeat/cmd" - - // register central management + // Register Fleet _ "github.com/elastic/beats/v7/x-pack/libbeat/management" - // Register fleet - _ "github.com/elastic/beats/v7/x-pack/libbeat/management/fleet" // register processors _ "github.com/elastic/beats/v7/x-pack/libbeat/processors/add_cloudfoundry_metadata" _ "github.com/elastic/beats/v7/x-pack/libbeat/processors/add_nomad_metadata" @@ -21,8 +17,3 @@ import ( _ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/elb" _ "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/nomad" ) - -// AddXPack extends the given root folder with XPack features -func AddXPack(root *cmd.BeatsRootCmd, name string) { - root.AddCommand(genEnrollCmd(name, "")) -} diff --git a/x-pack/libbeat/libbeat.go b/x-pack/libbeat/libbeat.go index eeaf82cccbd4..501787b36132 100644 --- a/x-pack/libbeat/libbeat.go +++ b/x-pack/libbeat/libbeat.go @@ -9,14 +9,13 @@ import ( "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/mock" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // RootCmd to test libbeat var RootCmd = cmd.GenRootCmdWithSettings(mock.New, mock.Settings) func main() { - xpackcmd.AddXPack(RootCmd, mock.Name) if err := RootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/x-pack/libbeat/management/api/auth_client.go b/x-pack/libbeat/management/api/auth_client.go deleted file mode 100644 index cb9f51139445..000000000000 --- a/x-pack/libbeat/management/api/auth_client.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "net/http" - "sort" - "time" - - "github.com/gofrs/uuid" - "github.com/joeshaw/multierror" -) - -// EventType is the type of event that the events endpoint can understand. -type EventType string - -// Event is the interface for the events to be send to the event endpoint. -type Event interface { - json.Marshaler - EventType() EventType -} - -// EventRequest is the data send to the CM event endpoint. -type EventRequest struct { - Timestamp time.Time `json:"timestamp"` - EventType EventType `json:"type"` - Event Event `json:"event"` -} - -// EventAPIResponse is the top level response for the events endpoints. -type EventAPIResponse struct { - BaseResponse - Response []EventResponse `json:"results"` -} - -// EventResponse is the indiviual response for each event request. -type EventResponse struct { - BaseResponse -} - -// AuthClienter is the interface exposed by the auth client and is useful for testing without calling -// a remote endpoint. -type AuthClienter interface { - // SendEvents takes a slices of event request and send them to the endpoint. - SendEvents([]EventRequest) error - - // Configuration retrieves the list of configuration blocks from Kibana - Configuration() (ConfigBlocks, error) -} - -// AuthClient is a authenticated client to the CM endpoint and exposes the calls that require -// the clients to pass credentials (UUID and AccessToken). -type AuthClient struct { - Client *Client - BeatUUID uuid.UUID - AccessToken string -} - -func (c AuthClient) headers() http.Header { - headers := http.Header{} - headers.Set("kbn-beats-access-token", c.AccessToken) - return headers -} - -// SendEvents send a list of events to Kibana. -func (c *AuthClient) SendEvents(requests []EventRequest) error { - sort.SliceStable(requests, func(i, j int) bool { - return requests[i].Timestamp.Before(requests[j].Timestamp) - }) - - resp := EventAPIResponse{} - url := fmt.Sprintf("/api/beats/%s/events", c.BeatUUID) - statusCode, err := c.Client.request("POST", url, requests, c.headers(), &resp) - if err != nil { - return err - } - - if statusCode != http.StatusOK { - return fmt.Errorf( - "invalid response code while sending events, expected 200 and received %d", - statusCode, - ) - } - - if len(resp.Response) != len(requests) { - return fmt.Errorf( - "number of response and the request do not match, expecting %d and received %d", - len(requests), - len(resp.Response), - ) - } - - // Loop through the responses and see if all items are marked as `success` we assume the response - // are in the same order as the sending order. - // - // We could add logic later to retry them, currently if sending error fails it's probably because - // Kibana is not answering and the next fetch will probably fails. - var errors multierror.Errors - for _, response := range resp.Response { - if !response.Success { - errors = append(errors, fmt.Errorf("error sending event, reason: %+v", response.Error.Message)) - } - } - - return errors.Err() -} diff --git a/x-pack/libbeat/management/api/auth_client_test.go b/x-pack/libbeat/management/api/auth_client_test.go deleted file mode 100644 index 340a98847e77..000000000000 --- a/x-pack/libbeat/management/api/auth_client_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "net/http" - "testing" - "time" - - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" -) - -var testEventType = EventType("TEST_EVENT") - -// Create a custom Event type for testing. -type testEvent struct { - Message string `json:"message"` - Type EventType `json:"event_type"` -} - -func (er *EventRequest) UnmarshalJSON(b []byte) error { - resp := struct { - EventType EventType `json:"type"` - Event json.RawMessage `json:"event"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - switch resp.EventType { - case testEventType: - event := &testEvent{} - if err := json.Unmarshal(resp.Event, event); err != nil { - return err - } - *er = EventRequest{EventType: resp.EventType, Event: event} - return nil - } - return fmt.Errorf("unknown event type of '%s'", resp.EventType) -} - -func (t *testEvent) EventType() EventType { - return t.Type -} -func (t *testEvent) MarshalJSON() ([]byte, error) { - return json.Marshal(*t) -} - -func (t *testEvent) UnmarshalJSON(b []byte) error { - resp := struct { - Message string `json:"message"` - Type EventType `json:"type"` - }{} - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - *t = testEvent{Message: resp.Message} - return nil -} - -func TestReportEvents(t *testing.T) { - beatUUID, err := uuid.NewV4() - if !assert.NoError(t, err) { - return - } - - accessToken := "my-enroll-token" - - t.Run("successfully send events", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/"+beatUUID.String()+"/events", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, accessToken, r.Header.Get("kbn-beats-access-token")) - - var response []EventRequest - - decoder := json.NewDecoder(r.Body) - err := decoder.Decode(&response) - if !assert.NoError(t, err) { - return - } - - if !assert.Equal(t, 1, len(response)) { - return - } - - expected := &testEvent{Message: "OK"} - received := response[0].Event.(*testEvent) - - if !assert.Equal(t, expected.Message, received.Message) { - return - } - - apiResponse := EventAPIResponse{ - Response: []EventResponse{EventResponse{BaseResponse: BaseResponse{Success: true}}}, - } - - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{&testEvent{Message: "OK"}} - - err = reportEvents(auth, events) - assert.NoError(t, err) - }) - - t.Run("bubble up any errors", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadRequest) - response := BaseResponse{ - Success: false, - Error: ErrorResponse{ - Message: "bad request", - Code: http.StatusBadRequest, - }, - } - json.NewEncoder(w).Encode(response) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{&testEvent{Message: "OK"}} - - err = reportEvents(auth, events) - assert.Error(t, err) - }) - - t.Run("assert the response", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiResponse := EventAPIResponse{ - Response: []EventResponse{ - EventResponse{BaseResponse: BaseResponse{Success: true}}, - EventResponse{BaseResponse: BaseResponse{Success: false}}, - }, - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{ - &testEvent{Message: "testing-1"}, - &testEvent{Message: "testing-2"}, - } - - err = reportEvents(auth, events) - assert.Error(t, err) - }) - - t.Run("enforce the match of response/request", func(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - apiResponse := EventAPIResponse{ - Response: []EventResponse{ - EventResponse{BaseResponse: BaseResponse{Success: true}}, - }, - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(apiResponse) - })) - defer server.Close() - - auth := &AuthClient{Client: client, AccessToken: accessToken, BeatUUID: beatUUID} - - events := []*testEvent{ - &testEvent{Message: "testing-1"}, - &testEvent{Message: "testing-2"}, - } - - err = reportEvents(auth, events) - assert.Error(t, err) - }) -} - -func reportEvents(client AuthClienter, events []*testEvent) error { - requests := make([]EventRequest, len(events)) - for idx, err := range events { - requests[idx] = EventRequest{ - Timestamp: time.Now(), - EventType: testEventType, - Event: err, - } - } - return client.SendEvents(requests) -} diff --git a/x-pack/libbeat/management/api/client.go b/x-pack/libbeat/management/api/client.go deleted file mode 100644 index 6212a69379f5..000000000000 --- a/x-pack/libbeat/management/api/client.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "bytes" - "encoding/json" - "net/http" - "net/url" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/kibana" -) - -const defaultTimeout = 10 * time.Second - -// Client to Central Management API -type Client struct { - client *kibana.Client -} - -// ConfigFromURL generates a full kibana client config from an URL -func ConfigFromURL(kibanaURL string, config *common.Config) (*kibana.ClientConfig, error) { - data, err := url.Parse(kibanaURL) - if err != nil { - return nil, err - } - - var username, password string - if data.User != nil { - username = data.User.Username() - password, _ = data.User.Password() - } - - // Lets pick up any configuration from either the YAML or from the -E flags. - // and merge it with the provided URL. - kibana := kibana.ClientConfig{} - if err := config.Unpack(&kibana); err != nil { - return nil, err - } - - kibana.Protocol = data.Scheme - kibana.Host = data.Host - kibana.Path = data.Path - kibana.Username = username - kibana.Password = password - kibana.Timeout = defaultTimeout - - return &kibana, nil -} - -// NewClient creates and returns a kibana client -func NewClient(cfg *kibana.ClientConfig) (*Client, error) { - client, err := kibana.NewClientWithConfig(cfg) - if err != nil { - return nil, err - } - return &Client{ - client: client, - }, nil -} - -// do a request to the API and unmarshall the message, error if anything fails -func (c *Client) request(method, extraPath string, - body interface{}, headers http.Header, resp interface{}) (int, error) { - - bodyJSON, err := json.Marshal(body) - if err != nil { - return 400, err - } - - statusCode, result, err := c.client.Request( - method, - extraPath, - nil, - headers, - bytes.NewBuffer(bodyJSON), - ) - if err != nil { - return statusCode, err - } - - if statusCode >= 300 { - err = extractError(result) - } else { - if err = json.Unmarshal(result, resp); err != nil { - return statusCode, errors.Wrap(err, "error unmarshaling Kibana response") - } - } - - return statusCode, err -} - -func extractError(b []byte) error { - var result BaseResponse - if err := json.Unmarshal(b, &result); err != nil { - return errors.Wrap(err, "error while parsing Kibana response") - } - return errors.New(result.Error.Message) -} diff --git a/x-pack/libbeat/management/api/client_test.go b/x-pack/libbeat/management/api/client_test.go deleted file mode 100644 index 0bdd8d2de59d..000000000000 --- a/x-pack/libbeat/management/api/client_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/elastic/beats/v7/libbeat/common" -) - -func newServerClientPair(t *testing.T, handler http.HandlerFunc) (*httptest.Server, *Client) { - mux := http.NewServeMux() - mux.Handle("/api/status", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "Unauthorized", 401) - })) - mux.Handle("/", handler) - - server := httptest.NewServer(mux) - - config, err := ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config.IgnoreVersion = true - - client, err := NewClient(config) - if err != nil { - t.Fatal(err) - } - - return server, client -} diff --git a/x-pack/libbeat/management/api/configuration.go b/x-pack/libbeat/management/api/configuration.go deleted file mode 100644 index 04bf525eab4f..000000000000 --- a/x-pack/libbeat/management/api/configuration.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "net/http" - "sort" - - "github.com/mitchellh/hashstructure" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common/reload" - - "github.com/elastic/beats/v7/libbeat/common" -) - -var errConfigurationNotFound = errors.New("no configuration found, you need to enroll your Beat") - -// ConfigBlock stores a piece of config from central management -type ConfigBlock struct { - Raw map[string]interface{} -} - -// ConfigBlocksWithType is a list of config blocks with the same type -type ConfigBlocksWithType struct { - Type string - Blocks []*ConfigBlock -} - -// ConfigBlocks holds a list of type + configs objects -type ConfigBlocks []ConfigBlocksWithType - -// Config returns a common.Config object holding the config from this block -func (c *ConfigBlock) Config() (*common.Config, error) { - return common.NewConfigFrom(c.Raw) -} - -// ConfigWithMeta returns a reload.ConfigWithMeta object holding the config from this block, meta will be nil -func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { - config, err := c.Config() - if err != nil { - return nil, err - } - return &reload.ConfigWithMeta{ - Config: config, - }, nil -} - -type configResponse struct { - Type string - Raw map[string]interface{} -} - -func (c *configResponse) UnmarshalJSON(b []byte) error { - var resp = struct { - Type string `json:"type"` - Raw map[string]interface{} `json:"config"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - converter := selectConverter(resp.Type) - newMap, err := converter(resp.Raw) - if err != nil { - return err - } - *c = configResponse{ - Type: resp.Type, - Raw: newMap, - } - return nil -} - -// Configuration retrieves the list of configuration blocks from Kibana -func (c *AuthClient) Configuration() (ConfigBlocks, error) { - resp := struct { - BaseResponse - ConfigBlocks []*configResponse `json:"list"` - }{} - url := fmt.Sprintf("/api/beats/agent/%s/configuration", c.BeatUUID) - statusCode, err := c.Client.request("GET", url, nil, c.headers(), &resp) - if statusCode == http.StatusNotFound { - return nil, errConfigurationNotFound - } - - if err != nil { - return nil, err - } - - blocks := map[string][]*ConfigBlock{} - for _, block := range resp.ConfigBlocks { - blocks[block.Type] = append(blocks[block.Type], &ConfigBlock{Raw: block.Raw}) - } - - // keep the ordering consistent while grouping the items. - keys := make([]string, 0, len(blocks)) - for k := range blocks { - keys = append(keys, k) - } - sort.Strings(keys) - - res := ConfigBlocks{} - for _, t := range keys { - b := blocks[t] - res = append(res, ConfigBlocksWithType{Type: t, Blocks: b}) - } - - return res, nil -} - -// ConfigBlocksEqual returns true if the given config blocks are equal, false if not -func ConfigBlocksEqual(a, b ConfigBlocks) (bool, error) { - // If there is an errors when hashing the config blocks its because the format changed. - aHash, err := hashstructure.Hash(a, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - bHash, err := hashstructure.Hash(b, nil) - if err != nil { - return false, errors.Wrap(err, "could not hash config blocks") - } - - return aHash == bHash, nil -} - -// IsConfigurationNotFound returns true if the configuration was not found. -func IsConfigurationNotFound(err error) bool { - return err == errConfigurationNotFound -} diff --git a/x-pack/libbeat/management/api/configuration_test.go b/x-pack/libbeat/management/api/configuration_test.go deleted file mode 100644 index 94fed11f17fd..000000000000 --- a/x-pack/libbeat/management/api/configuration_test.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "net/http" - "testing" - - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" -) - -// {"list":[{"id":"6c385a04-f315-489e-9208-c87f41911782","type":"filebeat.inputs","config":{"paths":["/tmp/hello.log"]},"tag":"89be4cfd-6249-4ac2-abe2-8f82520ba435"},{"id":"315ff7e9-ae24-4c99-a9d0-ed4314bc8e60","type":"output","config":{"_sub_type":"elasticsearch","username":"elastic","password":"changeme"},"tag":"89be4cfd-6249-4ac2-abe2-8f82520ba435"}],"success":true} -func TestConfiguration(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat ID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String()+"/configuration", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-access-token")) - - fmt.Fprintf(w, `{"success": true, "list":[{"type":"filebeat.modules","config":{"_sub_type":"apache2"}},{"type":"metricbeat.modules","config":{"_sub_type":"system","period":"10s"}}]}`) - })) - defer server.Close() - - auth := AuthClient{Client: client, AccessToken: "thisismyenrollmenttoken", BeatUUID: beatUUID} - - configs, err := auth.Configuration() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, 2, len(configs)) - checked := 0 - for _, config := range configs { - if config.Type == "metricbeat.modules" { - assert.Equal(t, &ConfigBlock{Raw: map[string]interface{}{ - "module": "system", - "period": "10s", - }}, config.Blocks[0]) - checked++ - - } else if config.Type == "filebeat.modules" { - assert.Equal(t, &ConfigBlock{Raw: map[string]interface{}{ - "module": "apache2", - }}, config.Blocks[0]) - checked++ - } - } - - assert.Equal(t, 2, checked) -} - -func TestConfigBlocksEqual(t *testing.T) { - tests := []struct { - name string - a, b ConfigBlocks - equal bool - }{ - { - name: "empty lists or nil", - a: nil, - b: ConfigBlocks{}, - equal: true, - }, - { - name: "single element", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: true, - }, - { - name: "single element with slices", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": []string{"foo", "bar"}, - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": []string{"foo", "bar"}, - }, - }, - }, - }, - }, - equal: true, - }, - { - name: "different number of blocks", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - &ConfigBlock{ - Raw: map[string]interface{}{ - "baz": "buzz", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: false, - }, - { - name: "different block", - a: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "baz": "buzz", - }, - }, - }, - }, - }, - b: ConfigBlocks{ - ConfigBlocksWithType{ - Type: "metricbeat.modules", - Blocks: []*ConfigBlock{ - &ConfigBlock{ - Raw: map[string]interface{}{ - "foo": "bar", - }, - }, - }, - }, - }, - equal: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - check, err := ConfigBlocksEqual(test.a, test.b) - if !assert.NoError(t, err) { - return - } - assert.Equal(t, test.equal, check) - }) - } -} - -func TestUnEnroll(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat UUID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String()+"/configuration", r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-access-token")) - - http.NotFound(w, r) - })) - defer server.Close() - - auth := AuthClient{Client: client, AccessToken: "thisismyenrollmenttoken", BeatUUID: beatUUID} - _, err = auth.Configuration() - assert.True(t, IsConfigurationNotFound(err)) -} diff --git a/x-pack/libbeat/management/api/convert.go b/x-pack/libbeat/management/api/convert.go deleted file mode 100644 index af7da53be468..000000000000 --- a/x-pack/libbeat/management/api/convert.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "strings" -) - -type converter func(map[string]interface{}) (map[string]interface{}, error) - -var mapper = map[string]converter{ - ".inputs": noopConvert, - ".modules": convertMultiple, - "output": convertSingle, -} - -var errSubTypeNotFound = fmt.Errorf("'%s' key not found", subTypeKey) - -var ( - subTypeKey = "_sub_type" - moduleKey = "module" -) - -func selectConverter(t string) converter { - for k, v := range mapper { - if strings.Index(t, k) > -1 { - return v - } - } - return noopConvert -} - -func convertSingle(m map[string]interface{}) (map[string]interface{}, error) { - subType, err := extractSubType(m) - if err != nil { - return nil, err - } - - delete(m, subTypeKey) - newMap := map[string]interface{}{subType: m} - return newMap, nil -} - -func convertMultiple(m map[string]interface{}) (map[string]interface{}, error) { - subType, err := extractSubType(m) - if err != nil { - return nil, err - } - - v, ok := m[moduleKey] - - if ok && v != subType { - return nil, fmt.Errorf("module key already exist in the raw document and doesn't match the 'sub_type', expecting '%s' and received '%s", subType, v) - } - - m[moduleKey] = subType - delete(m, subTypeKey) - return m, nil -} - -func noopConvert(m map[string]interface{}) (map[string]interface{}, error) { - return m, nil -} - -func extractSubType(m map[string]interface{}) (string, error) { - subType, ok := m[subTypeKey] - if !ok { - return "", errSubTypeNotFound - } - - k, ok := subType.(string) - if !ok { - return "", fmt.Errorf("invalid type for `sub_type`, expecting a string received %T", subType) - } - return k, nil -} diff --git a/x-pack/libbeat/management/api/convert_test.go b/x-pack/libbeat/management/api/convert_test.go deleted file mode 100644 index e8f32e90494e..000000000000 --- a/x-pack/libbeat/management/api/convert_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestConvertAPI(t *testing.T) { - tests := map[string]struct { - t string - config map[string]interface{} - expected map[string]interface{} - err bool - }{ - "output": { - t: "output", - config: map[string]interface{}{ - "_sub_type": "elasticsearch", - "username": "foobar", - }, - expected: map[string]interface{}{ - "elasticsearch": map[string]interface{}{ - "username": "foobar", - }, - }, - }, - "filebeat inputs": { - t: "filebeat.inputs", - config: map[string]interface{}{ - "type": "log", - "paths": []string{ - "/var/log/message.log", - "/var/log/system.log", - }, - }, - expected: map[string]interface{}{ - "type": "log", - "paths": []string{ - "/var/log/message.log", - "/var/log/system.log", - }, - }, - }, - "filebeat modules": { - t: "filebeat.modules", - config: map[string]interface{}{ - "_sub_type": "system", - }, - expected: map[string]interface{}{ - "module": "system", - }, - }, - "metricbeat modules": { - t: "metricbeat.modules", - config: map[string]interface{}{ - "_sub_type": "logstash", - }, - expected: map[string]interface{}{ - "module": "logstash", - }, - }, - "badly formed output": { - err: true, - t: "output", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "badly formed filebeat module": { - err: true, - t: "filebeat.modules", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "badly formed metricbeat module": { - err: true, - t: "metricbeat.modules", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - "unknown type is passthrough": { - t: "unkown", - config: map[string]interface{}{ - "nosubtype": "logstash", - }, - expected: map[string]interface{}{ - "nosubtype": "logstash", - }, - }, - } - - for name, test := range tests { - test := test - t.Run(name, func(t *testing.T) { - converter := selectConverter(test.t) - newMap, err := converter(test.config) - if !assert.Equal(t, test.err, err != nil) { - return - } - assert.True(t, reflect.DeepEqual(newMap, test.expected)) - }) - } -} diff --git a/x-pack/libbeat/management/api/doc.go b/x-pack/libbeat/management/api/doc.go deleted file mode 100644 index c0a95acfc8c9..000000000000 --- a/x-pack/libbeat/management/api/doc.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -/* -The Kibana CM Api returns a configuration format which cannot be ingested directly by our -configuration parser, it need to be transformed from the generic format into an adapted format -which is dependant on the type of configuration. - - -Translations: - -Type: output - -{ - "success": true, - "list": [ - - { - "config": { - "_sub_type": "elasticsearch" - "_id": "12312341231231" - "hosts": [ "localhost" ], - "password": "foobar" - "username": "elastic" - }, - "type": "output" - } - ] -} - -YAML representation: - -{ - "elasticsearch": { - "hosts": [ "localhost" ], - "password": "foobar" - "username": "elastic" - } -} - - -Type: *.modules - -{ - "success": true, - "list": [ - { - "config": { - "_sub_type": "system" - "_id": "12312341231231" - "path" "foobar" - }, - "type": "filebeat.module" - } - ] -} - -YAML representation: - -[ -{ - "module": "system" - "path": "foobar" -} -] - -*/ - -package api diff --git a/x-pack/libbeat/management/api/enroll.go b/x-pack/libbeat/management/api/enroll.go deleted file mode 100644 index 354d263d0760..000000000000 --- a/x-pack/libbeat/management/api/enroll.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "net/http" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" -) - -type enrollResponse struct { - BaseResponse - AccessToken string `json:"item"` -} - -func (e *enrollResponse) Validate() error { - if !e.Success || len(e.AccessToken) == 0 { - return errors.New("empty access_token") - } - return nil -} - -// Enroll a beat in central management, this call returns a valid access token to retrieve -// configurations -func (c *Client) Enroll( - beatType, beatName, beatVersion, hostname string, - beatUUID uuid.UUID, - enrollmentToken string, -) (string, error) { - params := common.MapStr{ - "type": beatType, - "name": beatName, - "version": beatVersion, - "host_name": hostname, - } - - resp := enrollResponse{} - - headers := http.Header{} - headers.Set("kbn-beats-enrollment-token", enrollmentToken) - - _, err := c.request("POST", "/api/beats/agent/"+beatUUID.String(), params, headers, &resp) - if err != nil { - return "", err - } - - if err := resp.Validate(); err != nil { - return "", err - } - - return resp.AccessToken, err -} diff --git a/x-pack/libbeat/management/api/enroll_test.go b/x-pack/libbeat/management/api/enroll_test.go deleted file mode 100644 index a120fb8a2e8a..000000000000 --- a/x-pack/libbeat/management/api/enroll_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "testing" - - "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" -) - -func TestEnrollValid(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating Beat ID: %v", err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := ioutil.ReadAll(r.Body) - if err != nil { - t.Fatal(err) - } - - // Check correct path is used - assert.Equal(t, "/api/beats/agent/"+beatUUID.String(), r.URL.Path) - - // Check enrollment token is correct - assert.Equal(t, "thisismyenrollmenttoken", r.Header.Get("kbn-beats-enrollment-token")) - - request := struct { - Hostname string `json:"host_name"` - Type string `json:"type"` - Version string `json:"version"` - Name string `json:"name"` - }{} - if err := json.Unmarshal(body, &request); err != nil { - t.Fatal(err) - } - - assert.Equal(t, "myhostname.lan", request.Hostname) - assert.Equal(t, "metricbeat", request.Type) - assert.Equal(t, "6.3.0", request.Version) - assert.Equal(t, "beatname", request.Name) - - fmt.Fprintf(w, `{"success": true, "item": "fooo"}`) - })) - defer server.Close() - - accessToken, err := client.Enroll("metricbeat", "beatname", "6.3.0", "myhostname.lan", beatUUID, "thisismyenrollmenttoken") - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "fooo", accessToken) -} - -func TestEnrollError(t *testing.T) { - tests := map[string]struct { - body string - responseCode int - }{ - "invalid enrollment token": { - body: `{"success": false, "error": { "message": "Invalid enrollment token", "code": 400 }}`, - responseCode: 400, - }, - //NOTE(ph): I believe this is now fixed in the API. - "invalid token response": { - body: `{"success": true, "item": ""}`, - responseCode: 200, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - beatUUID, err := uuid.NewV4() - if err != nil { - t.Fatal(err) - } - - server, client := newServerClientPair(t, http.HandlerFunc(func( - w http.ResponseWriter, - r *http.Request, - ) { - http.Error(w, test.body, test.responseCode) - })) - defer server.Close() - - accessToken, err := client.Enroll( - "metricbeat", - "beatname", - "6.3.0", - "myhostname.lan", - beatUUID, - "thisismyenrollmenttoken", - ) - - assert.Error(t, err) - assert.Equal(t, "", accessToken) - }) - } -} diff --git a/x-pack/libbeat/management/api/enrollment_token.go b/x-pack/libbeat/management/api/enrollment_token.go deleted file mode 100644 index cf20a075a736..000000000000 --- a/x-pack/libbeat/management/api/enrollment_token.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "net/http" -) - -// CreateEnrollmentToken talks to Kibana API and generates an enrollment token -func (c *Client) CreateEnrollmentToken() (string, error) { - headers := http.Header{} - - resp := struct { - Results []struct { - Token string `json:"item"` - } `json:"results"` - }{} - - _, err := c.request("POST", "/api/beats/enrollment_tokens", nil, headers, &resp) - if err != nil { - return "", err - } - - if tokensCount := len(resp.Results); tokensCount != 1 { - return "", fmt.Errorf("Unexpected number of tokens, got %d, only one expected", tokensCount) - } - - return resp.Results[0].Token, nil -} diff --git a/x-pack/libbeat/management/api/enrollment_token_test.go b/x-pack/libbeat/management/api/enrollment_token_test.go deleted file mode 100644 index 2b5241e8f31b..000000000000 --- a/x-pack/libbeat/management/api/enrollment_token_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestEnrollmentToken(t *testing.T) { - server, client := newServerClientPair(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Check correct path is used - assert.Equal(t, "/api/beats/enrollment_tokens", r.URL.Path) - fmt.Fprintf(w, `{"results": [{"item":"65074ff8639a4661ba7e1bd5ccc209ed"}]}`) - })) - defer server.Close() - - token, err := client.CreateEnrollmentToken() - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "65074ff8639a4661ba7e1bd5ccc209ed", token) -} diff --git a/x-pack/libbeat/management/api/event_reporter.go b/x-pack/libbeat/management/api/event_reporter.go deleted file mode 100644 index 8ed492c7e6b0..000000000000 --- a/x-pack/libbeat/management/api/event_reporter.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "sync" - "time" - - "github.com/joeshaw/multierror" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -const debugK = "event_reporter" - -// EventReporter is an object that will periodically send asyncronously events to the -// CM events endpoints. -type EventReporter struct { - logger *logp.Logger - client AuthClienter - period time.Duration - maxBatchSize int - done chan struct{} - buffer []Event - mu sync.Mutex - wg sync.WaitGroup -} - -// NewEventReporter returns a new event reporter -func NewEventReporter( - logger *logp.Logger, - client AuthClienter, - period time.Duration, - maxBatchSize int, -) *EventReporter { - log := logger.Named(debugK) - return &EventReporter{ - logger: log, - client: client, - period: period, - maxBatchSize: maxBatchSize, - done: make(chan struct{}), - } -} - -// Start starts the event reported and wait for new events. -func (e *EventReporter) Start() { - e.wg.Add(1) - go e.worker() - e.logger.Info("Starting event reporter service") -} - -// Stop stops the reporting events to the endpoint. -func (e *EventReporter) Stop() { - e.logger.Info("Stopping event reporter service") - close(e.done) - e.wg.Wait() -} - -func (e *EventReporter) worker() { - defer e.wg.Done() - ticker := time.NewTicker(e.period) - defer ticker.Stop() - - var done bool - for !done { - select { - case <-e.done: - done = true - case <-ticker.C: - } - - var buf []Event - e.mu.Lock() - buf, e.buffer = e.buffer, nil - e.mu.Unlock() - - e.reportEvents(buf) - } -} - -func (e *EventReporter) reportEvents(events []Event) { - if len(events) == 0 { - return - } - e.logger.Debugf("Reporting %d events to Kibana", len(events)) - if err := e.sendBatchEvents(events); err != nil { - e.logger.Errorf("could not send events, error: %+v", err) - } -} - -func (e *EventReporter) sendBatchEvents(events []Event) error { - var errors multierror.Errors - for pos := 0; pos < len(events); pos += e.maxBatchSize { - j := pos + e.maxBatchSize - if j > len(events) { - j = len(events) - } - if err := e.sendEvents(events[pos:j]); err != nil { - errors = append(errors, err) - } - } - return errors.Err() -} - -func (e *EventReporter) sendEvents(events []Event) error { - requests := make([]EventRequest, len(events)) - for i, event := range events { - requests[i] = EventRequest{ - Timestamp: time.Now(), - EventType: event.EventType(), - Event: event, - } - } - return e.client.SendEvents(requests) -} - -// AddEvent adds an event to be send on the next tick. -func (e *EventReporter) AddEvent(event Event) { - e.mu.Lock() - defer e.mu.Unlock() - e.buffer = append(e.buffer, event) -} diff --git a/x-pack/libbeat/management/api/event_reporter_test.go b/x-pack/libbeat/management/api/event_reporter_test.go deleted file mode 100644 index e1576bbd0934..000000000000 --- a/x-pack/libbeat/management/api/event_reporter_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import ( - "math" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/libbeat/logp" -) - -type memoryAuthClient struct { - Requests chan []EventRequest - Err error -} - -func (m *memoryAuthClient) SendEvents(requests []EventRequest) error { - if m.Err != nil { - return m.Err - } - - m.Requests <- requests - return nil -} - -func (m *memoryAuthClient) Close() { - close(m.Requests) -} - -func (m *memoryAuthClient) Configuration() (ConfigBlocks, error) { - return ConfigBlocks{}, nil -} - -func newMemoryAuthClient() *memoryAuthClient { - return &memoryAuthClient{Requests: make(chan []EventRequest)} -} - -func TestReporterReportEvents(t *testing.T) { - t.Run("single request", testBatch(1, 100)) - t.Run("receive all events when the requests size is exactly the batch size", testBatch(100, 100)) - t.Run("receive all events when events are send in multiple batch", testBatch(1234, 25)) -} - -func testBatch(numberOfEvents, maxBatchSize int) func(*testing.T) { - return func(t *testing.T) { - event := &testEvent{Message: "OK"} - client := newMemoryAuthClient() - defer client.Close() - reporter := NewEventReporter(logp.NewLogger(""), client, 1*time.Second, maxBatchSize) - reporter.Start() - defer reporter.Stop() - - go func() { - for i := 0; i < numberOfEvents; i++ { - reporter.AddEvent(event) - } - }() - - var receivedEvents int - expectedbatch := int(math.Ceil(float64(numberOfEvents) / float64(maxBatchSize))) - - for receivedBatchs := 0; receivedBatchs < expectedbatch; receivedBatchs++ { - requests := <-client.Requests - receivedEvents += len(requests) - } - - assert.Equal(t, numberOfEvents, receivedEvents) - } -} diff --git a/x-pack/libbeat/management/api/response.go b/x-pack/libbeat/management/api/response.go deleted file mode 100644 index cb28924bb250..000000000000 --- a/x-pack/libbeat/management/api/response.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package api - -import "fmt" - -// Action are the actions executed on the API. -type Action int - -// List of the valid Actions executed by the API. -//go:generate stringer -type=LicenseType -linecomment=true -const ( - Created Action = iota + 1 // created -) - -var mapStringToAction = map[string]Action{ - "created": Created, -} - -// UnmarshalJSON unmarshal an action string into a constant. -func (a *Action) UnmarshalJSON(b []byte) error { - k := string(b) - if len(b) <= 2 { - return fmt.Errorf( - "invalid string for action type, received: '%s'", - k, - ) - } - v, found := mapStringToAction[k[1:len(k)-1]] - if !found { - return fmt.Errorf( - "unknown action '%s' returned from the API, valid actions are: 'created'", - k, - ) - } - *a = v - return nil -} - -// BaseResponse the common response from all the API calls. -type BaseResponse struct { - Action Action `json:"action,omitempty"` - Success bool `json:"success"` - Error ErrorResponse `json:"error,omitempty"` -} - -// ErrorResponse contains human readable and machine readable information when an error happens. -type ErrorResponse struct { - Message string `json:"message"` - Code int `json:"code"` -} diff --git a/x-pack/libbeat/management/blacklist.go b/x-pack/libbeat/management/blacklist.go index 9e3326aa537f..6ea5088a962d 100644 --- a/x-pack/libbeat/management/blacklist.go +++ b/x-pack/libbeat/management/blacklist.go @@ -8,11 +8,11 @@ import ( "fmt" "strings" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/match" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) // ConfigBlacklist takes a ConfigBlocks object and filter it based on the given @@ -61,22 +61,19 @@ func NewConfigBlacklist(cfg ConfigBlacklistSettings) (*ConfigBlacklist, error) { } // Detect an error if any of the given config blocks is blacklisted -func (c *ConfigBlacklist) Detect(configBlocks api.ConfigBlocks) Errors { - var errs Errors +func (c *ConfigBlacklist) Detect(configBlocks ConfigBlocks) error { + var err *multierror.Error for _, configs := range configBlocks { for _, block := range configs.Blocks { if c.isBlacklisted(configs.Type, block) { - errs = append(errs, &Error{ - Type: ConfigError, - Err: fmt.Errorf("Config for '%s' is blacklisted", configs.Type), - }) + err = multierror.Append(err, fmt.Errorf("Config for '%s' is blacklisted", configs.Type)) } } } - return errs + return err.ErrorOrNil() } -func (c *ConfigBlacklist) isBlacklisted(blockType string, block *api.ConfigBlock) bool { +func (c *ConfigBlacklist) isBlacklisted(blockType string, block *ConfigBlock) bool { cfg, err := block.ConfigWithMeta() if err != nil { return false diff --git a/x-pack/libbeat/management/blacklist_test.go b/x-pack/libbeat/management/blacklist_test.go index 4bd2e36352e8..ead5060d9612 100644 --- a/x-pack/libbeat/management/blacklist_test.go +++ b/x-pack/libbeat/management/blacklist_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" ) func TestConfigBlacklistSettingsUnpack(t *testing.T) { @@ -67,16 +66,16 @@ func TestConfigBlacklist(t *testing.T) { tests := []struct { name string patterns map[string]string - blocks api.ConfigBlocks + blocks ConfigBlocks blacklisted bool }{ { name: "No patterns", blacklisted: false, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "output", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "output": "console", @@ -92,10 +91,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "output": "^console$", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "output", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "console": map[string]interface{}{ @@ -120,10 +119,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "metricbeat.modules.module": "k.{8}s", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -140,10 +139,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "metricbeat.modules.metricsets": "event", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -171,10 +170,10 @@ func TestConfigBlacklist(t *testing.T) { patterns: map[string]string{ "filebeat.inputs.containers.ids": "1ffeb0dbd13", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "metricbeat.modules", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "module": "kubernetes", @@ -186,9 +185,9 @@ func TestConfigBlacklist(t *testing.T) { }, }, }, - api.ConfigBlocksWithType{ + ConfigBlocksWithType{ Type: "filebeat.inputs", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "type": "docker", @@ -220,10 +219,10 @@ func TestConfigBlacklist(t *testing.T) { "list.of.elements": "forbidden", "list.of.elements.disallowed": "yes", }, - blocks: api.ConfigBlocks{ - api.ConfigBlocksWithType{ + blocks: ConfigBlocks{ + ConfigBlocksWithType{ Type: "list", - Blocks: []*api.ConfigBlock{ + Blocks: []*ConfigBlock{ { Raw: map[string]interface{}{ "of": map[string]interface{}{ @@ -274,7 +273,11 @@ func TestConfigBlacklist(t *testing.T) { } errs := bl.Detect(test.blocks) - assert.Equal(t, test.blacklisted, !errs.IsEmpty()) + if test.blacklisted { + assert.NotNil(t, errs) + } else { + assert.Nil(t, errs) + } }) } } diff --git a/x-pack/libbeat/management/cache.go b/x-pack/libbeat/management/cache.go deleted file mode 100644 index 58146a3cbe4e..000000000000 --- a/x-pack/libbeat/management/cache.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "io/ioutil" - "os" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/file" - "github.com/elastic/beats/v7/libbeat/paths" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" - - "github.com/pkg/errors" - "gopkg.in/yaml.v2" -) - -// Cache keeps a copy of configs provided by Kibana, it's used when Kibana is down -type Cache struct { - Configs api.ConfigBlocks -} - -// Load settings from its source file -func (c *Cache) Load() error { - path := paths.Resolve(paths.Data, "management.yml") - config, err := common.LoadFile(path) - if err != nil { - if os.IsNotExist(err) { - // File is not present, beat is not enrolled - return nil - } - return err - } - - if err = config.Unpack(&c); err != nil { - return err - } - - return nil -} - -// Save settings to management.yml file -func (c *Cache) Save() error { - path := paths.Resolve(paths.Data, "management.yml") - - data, err := yaml.Marshal(c) - if err != nil { - return err - } - - // write temporary file first - tempFile := path + ".new" - if err := ioutil.WriteFile(tempFile, data, 0600); err != nil { - return errors.Wrap(err, "failed to store central management settings") - } - - // move temporary file into final location - return file.SafeFileRotate(path, tempFile) -} - -// HasConfig returns true if configs are cached. -func (c *Cache) HasConfig() bool { - return len(c.Configs) > 0 -} diff --git a/x-pack/libbeat/management/cache_test.go b/x-pack/libbeat/management/cache_test.go deleted file mode 100644 index 1acbe76d60d0..000000000000 --- a/x-pack/libbeat/management/cache_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -func TestHasConfig(t *testing.T) { - tests := map[string]struct { - configs api.ConfigBlocks - expected bool - }{ - "with config": { - configs: api.ConfigBlocks{ - api.ConfigBlocksWithType{Type: "metricbeat "}, - }, - expected: true, - }, - "without config": { - configs: api.ConfigBlocks{}, - expected: false, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - cache := Cache{Configs: test.configs} - assert.Equal(t, test.expected, cache.HasConfig()) - }) - } -} diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index 61cb0c5cda43..cf285d5389d2 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -5,112 +5,32 @@ package management import ( - "io" - "text/template" - "time" - - "gopkg.in/yaml.v2" - - "github.com/elastic/beats/v7/libbeat/kibana" -) - -// ManagedConfigTemplate is used to overwrite settings file during enrollment -const ManagedConfigTemplate = ` - -#========================= Central Management ================================= - -# Beats is configured under central management, you can define most settings -# from the Kibana UI. You can update this file to configure the settings that -# are not supported by Kibana Beats management. - -{{.CentralManagementSettings}} -#================================ General ===================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. -#fields: -# env: staging - -#================================ Logging ===================================== - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: debug - -# At debug level, you can selectively enable logging only for some components. -# To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publisher", "service". -#logging.selectors: ["*"] - -#============================== X-Pack Monitoring =============================== -# {{.BeatName}} can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#monitoring.enabled: false - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. -# Note that the settings should point to your Elasticsearch *monitoring* cluster. -# Any setting that is not set is automatically inherited from the Elasticsearch -# output configuration, so if you have the Elasticsearch output configured such -# that it is pointing to your Elasticsearch monitoring cluster, you can simply -# uncomment the following line. -#monitoring.elasticsearch: -` - -const ( - // ModeCentralManagement is a default CM mode, using existing processes. - ModeCentralManagement = "x-pack-cm" - - // ModeFleet is a management mode where fleet is used to retrieve configurations. - ModeFleet = "x-pack-fleet" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/reload" ) -// Config for central management. +// Config for central management type Config struct { - // true when enrolled - Enabled bool `config:"enabled" yaml:"enabled"` - - // Mode specifies whether beat uses Central Management or Fleet. - // Options: [cm, fleet] - Mode string `config:"mode" yaml:"mode"` - - // Poll configs period - Period time.Duration `config:"period" yaml:"period"` - - EventsReporter EventReporterConfig `config:"events_reporter" yaml:"events_reporter"` - - AccessToken string `config:"access_token" yaml:"access_token"` - - Kibana *kibana.ClientConfig `config:"kibana" yaml:"kibana"` - + Enabled bool `config:"enabled" yaml:"enabled"` Blacklist ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` } -// EventReporterConfig configuration of the events reporter. -type EventReporterConfig struct { - Period time.Duration `config:"period" yaml:"period"` - MaxBatchSize int `config:"max_batch_size" yaml:"max_batch_size" validate:"nonzero,positive"` +// ConfigBlock stores a piece of config from central management +type ConfigBlock struct { + Raw map[string]interface{} +} + +// ConfigBlocksWithType is a list of config blocks with the same type +type ConfigBlocksWithType struct { + Type string + Blocks []*ConfigBlock } +// ConfigBlocks holds a list of type + configs objects +type ConfigBlocks []ConfigBlocksWithType + func defaultConfig() *Config { return &Config{ - Mode: ModeCentralManagement, - Period: 60 * time.Second, - EventsReporter: EventReporterConfig{ - Period: 30 * time.Second, - MaxBatchSize: 1000, - }, Blacklist: ConfigBlacklistSettings{ Patterns: map[string]string{ "output": "console|file", @@ -119,30 +39,18 @@ func defaultConfig() *Config { } } -type templateParams struct { - CentralManagementSettings string - BeatName string +// Config returns a common.Config object holding the config from this block +func (c *ConfigBlock) Config() (*common.Config, error) { + return common.NewConfigFrom(c.Raw) } -// OverwriteConfigFile will overwrite beat settings file with the enrolled template. -func (c *Config) OverwriteConfigFile(wr io.Writer, beatName string) error { - t := template.Must(template.New("beat.management.yml").Parse(ManagedConfigTemplate)) - - tmp := struct { - Management *Config `yaml:"management"` - }{ - Management: c, - } - - data, err := yaml.Marshal(tmp) +// ConfigWithMeta returns a reload.ConfigWithMeta object holding the config from this block, meta will be nil +func (c *ConfigBlock) ConfigWithMeta() (*reload.ConfigWithMeta, error) { + config, err := c.Config() if err != nil { - return err - } - - params := templateParams{ - CentralManagementSettings: string(data), - BeatName: beatName, + return nil, err } - - return t.Execute(wr, params) + return &reload.ConfigWithMeta{ + Config: config, + }, nil } diff --git a/x-pack/libbeat/management/config_test.go b/x-pack/libbeat/management/config_test.go deleted file mode 100644 index 25ece2e216c4..000000000000 --- a/x-pack/libbeat/management/config_test.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func EnsureBlacklistItems(t *testing.T) { - // NOTE: We do not permit to configure the console or the file output with CM for security reason. - c := defaultConfig() - v, _ := c.Blacklist.Patterns["output"] - assert.Equal(t, "console|file", v) -} diff --git a/x-pack/libbeat/management/enroll.go b/x-pack/libbeat/management/enroll.go deleted file mode 100644 index 407808ec7aed..000000000000 --- a/x-pack/libbeat/management/enroll.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "fmt" - "os" - "time" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/common/file" - "github.com/elastic/beats/v7/libbeat/keystore" - "github.com/elastic/beats/v7/libbeat/kibana" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -const accessTokenKey = "management.accesstoken" - -// Enroll this beat to the given kibana -// This will use Central Management API to enroll and retrieve an access key for config retrieval -func Enroll( - beat *instance.Beat, - kibanaConfig *kibana.ClientConfig, - enrollmentToken string, -) error { - // Ignore kibana version to avoid permission errors - kibanaConfig.IgnoreVersion = true - - client, err := api.NewClient(kibanaConfig) - if err != nil { - return err - } - - cfgwarn.Deprecate("8.0.0", "Central Management is no longer under development and has been deprecated. We are working hard to deliver a new and more comprehensive solution and look forward to sharing it with you") - - accessToken, err := client.Enroll(beat.Info.Beat, beat.Info.Name, beat.Info.Version, beat.Info.Hostname, beat.Info.ID, enrollmentToken) - if err != nil { - return err - } - - // Store access token in keystore - if err := storeAccessToken(beat, accessToken); err != nil { - return err - } - - // Enrolled, persist state - config := defaultConfig() - config.Enabled = true - config.AccessToken = "${" + accessTokenKey + "}" - config.Kibana = kibanaConfig - - configFile := cfgfile.GetDefaultCfgfile() - - ts := time.Now() - - // This timestamp format is a variation of RFC3339 replacing colons with - // slashes so that it can be used as part of a filename in all OSes. - // (Colon is not a valid character for filenames in Windows). - // Also removed the TZ-offset as that can cause a plus sign to be output. - const fsSafeTimestamp = "2006-01-02T15-04-05" - - // backup current settings: - backConfigFile := configFile + "." + ts.Format(fsSafeTimestamp) + ".bak" - fmt.Println("Saving a copy of current settings to " + backConfigFile) - err = file.SafeFileRotate(backConfigFile, configFile) - if err != nil { - return errors.Wrap(err, "creating a backup copy of current settings") - } - - // create the new ones: - f, err := os.OpenFile(configFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - return errors.Wrap(err, "opening settings file") - } - defer f.Close() - - if err := config.OverwriteConfigFile(f, beat.Beat.Info.Beat); err != nil { - return errors.Wrap(err, "overriding settings file") - } - - return nil -} - -func storeAccessToken(beat *instance.Beat, accessToken string) error { - keyStore := beat.Keystore() - - wKeystore, err := keystore.AsWritableKeystore(keyStore) - if err != nil { - return err - } - - if !keyStore.IsPersisted() { - - if err := wKeystore.Create(false); err != nil { - return errors.Wrap(err, "error creating keystore") - } - } - if err := wKeystore.Store(accessTokenKey, []byte(accessToken)); err != nil { - return errors.Wrap(err, "error storing the access token") - } - - return wKeystore.Save() -} diff --git a/x-pack/libbeat/management/error.go b/x-pack/libbeat/management/error.go deleted file mode 100644 index 31fd8df3d08a..000000000000 --- a/x-pack/libbeat/management/error.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "strconv" - "strings" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -// ErrorType is type of error that the events endpoint understand. -type ErrorType string - -// ConfigError is the type of error send when an unpack or a blacklist happen. -var ConfigError = ErrorType("CONFIG") - -// ErrorEvent is the event type when an error happen. -var ErrorEvent = api.EventType("ERROR") - -// Error is a config error to be reported to kibana. -type Error struct { - Type ErrorType - UUID uuid.UUID - Err error -} - -// EventType returns a ErrorEvent. -func (e *Error) EventType() api.EventType { - return ErrorEvent -} - -// MarshalJSON transform an error into a JSON document. -func (e *Error) MarshalJSON() ([]byte, error) { - return json.Marshal(&struct { - UUID string `json:"uuid"` - Type string `json:"type"` - Message string `json:"message"` - }{ - UUID: e.UUID.String(), - Type: string(e.Type), - Message: e.Err.Error(), - }) -} - -// UnmarshalJSON unmarshals a event of the type Error. -func (e *Error) UnmarshalJSON(b []byte) error { - res := &struct { - UUID string `json:"uuid,omitempty"` - Type string `json:"type"` - Message string `json:"message"` - }{} - - if err := json.Unmarshal(b, res); err != nil { - return err - } - - uuid, err := uuid.FromString(res.UUID) - if err != nil { - return err - } - *e = Error{ - Type: ErrorType(res.Type), - UUID: uuid, - Err: errors.New(res.Message), - } - return nil -} - -func (e *Error) Error() string { - return e.Err.Error() -} - -// Errors contains mutiples config error. -type Errors []*Error - -// Errors makes sure we can display the error in the logger. -func (er *Errors) Error() string { - var s strings.Builder - if len(*er) == 1 { - s.WriteString("1 error: ") - } else { - s.WriteString(strconv.Itoa(len(*er))) - s.WriteString(" errors: ") - } - for idx, err := range *er { - if idx != 0 { - s.WriteString("; ") - } - s.WriteString(err.Error()) - } - return s.String() -} - -// IsEmpty returns true when we don't have any errors. -func (er *Errors) IsEmpty() bool { - return len(*er) == 0 -} - -// NewConfigError wraps an error to be a management error of a specific ConfigError Type -func NewConfigError(err error) *Error { - return &Error{Type: ConfigError, Err: err} -} diff --git a/x-pack/libbeat/management/error_test.go b/x-pack/libbeat/management/error_test.go deleted file mode 100644 index 4d29d3aacad5..000000000000 --- a/x-pack/libbeat/management/error_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "testing" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -func TestErrorSerialization(t *testing.T) { - id, _ := uuid.NewV4() - t.Run("serialize ok", func(t *testing.T) { - e := Error{ - Type: ConfigError, - Err: errors.New("hello world"), - UUID: id, - } - - b, err := json.Marshal(&e) - if assert.NoError(t, err) { - return - } - - resp := &struct { - UUID string `json:"uuid"` - Message string `json:"message"` - Type string `json:"type"` - }{} - - err = json.Unmarshal(b, resp) - if assert.NoError(t, err) { - return - } - - assert.Equal(t, e.UUID.String(), resp.UUID) - assert.Equal(t, e.Err.Error(), resp.Message) - assert.Equal(t, e.Type, api.EventType(resp.Type)) - }) - - t.Run("ensure that json general fields are present", ensureJSONhasGeneralfield(t, &Error{ - Type: ConfigError, - Err: errors.New("hello world"), - UUID: id, - })) -} - -func TestErrors(t *testing.T) { - t.Run("single error", func(t *testing.T) { - errors := Errors{NewConfigError(errors.New("error1"))} - assert.Equal(t, "1 error: error1", errors.Error()) - }) - - t.Run("multiple errors", func(t *testing.T) { - errors := Errors{ - NewConfigError(errors.New("error1")), - NewConfigError(errors.New("error2")), - } - assert.Equal(t, "2 errors: error1; error2", errors.Error()) - }) -} diff --git a/x-pack/libbeat/management/fleet/config.go b/x-pack/libbeat/management/fleet/config.go deleted file mode 100644 index 9572d30acc9c..000000000000 --- a/x-pack/libbeat/management/fleet/config.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" -) - -// Config for central management -type Config struct { - Enabled bool `config:"enabled" yaml:"enabled"` - Mode string `config:"mode" yaml:"mode"` - Blacklist xmanagement.ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` -} - -func defaultConfig() *Config { - return &Config{ - Mode: xmanagement.ModeCentralManagement, - Blacklist: xmanagement.ConfigBlacklistSettings{ - Patterns: map[string]string{ - "output": "console|file", - }, - }, - } -} diff --git a/x-pack/libbeat/management/fleet/manager.go b/x-pack/libbeat/management/fleet/manager.go deleted file mode 100644 index 8ec84c10579f..000000000000 --- a/x-pack/libbeat/management/fleet/manager.go +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - "context" - "fmt" - "os" - "sort" - "sync" - - "github.com/gofrs/uuid" - "github.com/pkg/errors" - - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/management" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" - - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" -) - -// Manager handles internal config updates. By retrieving -// new configs from Kibana and applying them to the Beat. -type Manager struct { - config *Config - logger *logp.Logger - beatUUID uuid.UUID - registry *reload.Registry - blacklist *xmanagement.ConfigBlacklist - client client.Client - lock sync.Mutex - status management.Status - msg string - payload map[string]interface{} - - stopFunc func() -} - -// NewFleetManager returns a X-Pack Beats Fleet Management manager. -func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - c := defaultConfig() - if config.Enabled() { - if err := config.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "parsing fleet management settings") - } - } - return NewFleetManagerWithConfig(c, registry, beatUUID) -} - -// NewFleetManagerWithConfig returns a X-Pack Beats Fleet Management manager. -func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - log := logp.NewLogger(management.DebugK) - - m := &Manager{ - config: c, - logger: log.Named("fleet"), - beatUUID: beatUUID, - registry: registry, - } - - var err error - var blacklist *xmanagement.ConfigBlacklist - var eac client.Client - if c.Enabled && c.Mode == xmanagement.ModeFleet { - // Initialize configs blacklist - blacklist, err = xmanagement.NewConfigBlacklist(c.Blacklist) - if err != nil { - return nil, errors.Wrap(err, "wrong settings for configurations blacklist") - } - - // Initialize the client - eac, err = client.NewFromReader(os.Stdin, m) - if err != nil { - return nil, errors.Wrap(err, "failed to create elastic-agent-client") - } - } - - m.blacklist = blacklist - m.client = eac - return m, nil -} - -// Enabled returns true if config management is enabled. -func (cm *Manager) Enabled() bool { - return cm.config.Enabled && cm.config.Mode == xmanagement.ModeFleet -} - -// Start the config manager -func (cm *Manager) Start(stopFunc func()) { - if !cm.Enabled() { - return - } - - cfgwarn.Beta("Fleet management is enabled") - cm.logger.Info("Starting fleet management service") - - cm.stopFunc = stopFunc - err := cm.client.Start(context.Background()) - if err != nil { - cm.logger.Errorf("failed to start elastic-agent-client: %s", err) - } -} - -// Stop the config manager -func (cm *Manager) Stop() { - if !cm.Enabled() { - return - } - - cm.logger.Info("Stopping fleet management service") - cm.client.Stop() -} - -// CheckRawConfig check settings are correct to start the beat. This method -// checks there are no collision between the existing configuration and what -// fleet management can configure. -func (cm *Manager) CheckRawConfig(cfg *common.Config) error { - // TODO implement this method - return nil -} - -// UpdateStatus updates the manager with the current status for the beat. -func (cm *Manager) UpdateStatus(status management.Status, msg string) { - cm.lock.Lock() - defer cm.lock.Unlock() - - if cm.status != status || cm.msg != msg { - cm.status = status - cm.msg = msg - cm.client.Status(statusToProtoStatus(status), msg, nil) - cm.logger.Infof("Status change to %s: %s", status, msg) - } -} - -func (cm *Manager) OnConfig(s string) { - cm.UpdateStatus(management.Configuring, "Updating configuration") - - var configMap common.MapStr - uconfig, err := common.NewConfigFrom(s) - if err != nil { - err = errors.Wrap(err, "config blocks unsuccessfully generated") - cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) - return - } - - err = uconfig.Unpack(&configMap) - if err != nil { - err = errors.Wrap(err, "config blocks unsuccessfully generated") - cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) - return - } - - blocks, err := cm.toConfigBlocks(configMap) - if err != nil { - err = errors.Wrap(err, "failed to parse configuration") - cm.logger.Error(err) - cm.UpdateStatus(management.Failed, err.Error()) - return - } - - if errs := cm.apply(blocks); !errs.IsEmpty() { - // `cm.apply` already logs the errors; currently allow beat to run degraded - cm.UpdateStatus(management.Failed, errs.Error()) - return - } - - cm.client.Status(proto.StateObserved_HEALTHY, "Running", cm.payload) -} - -func (cm *Manager) RegisterAction(action client.Action) { - cm.client.RegisterAction(action) -} - -func (cm *Manager) UnregisterAction(action client.Action) { - cm.client.UnregisterAction(action) -} - -func (cm *Manager) SetPayload(payload map[string]interface{}) { - cm.lock.Lock() - cm.payload = payload - cm.lock.Unlock() -} - -func (cm *Manager) OnStop() { - if cm.stopFunc != nil { - cm.client.Status(proto.StateObserved_STOPPING, "Stopping", nil) - cm.stopFunc() - } -} - -func (cm *Manager) OnError(err error) { - cm.logger.Errorf("elastic-agent-client got error: %s", err) -} - -func (cm *Manager) apply(blocks api.ConfigBlocks) xmanagement.Errors { - var errors xmanagement.Errors - missing := map[string]bool{} - for _, name := range cm.registry.GetRegisteredNames() { - missing[name] = true - } - - // Detect unwanted configs from the list - if errs := cm.blacklist.Detect(blocks); !errs.IsEmpty() { - errors = append(errors, errs...) - return errors - } - - // Reload configs - for _, b := range blocks { - if err := cm.reload(b.Type, b.Blocks); err != nil { - errors = append(errors, err) - } - missing[b.Type] = false - } - - // Unset missing configs - for name := range missing { - if missing[name] { - if err := cm.reload(name, []*api.ConfigBlock{}); err != nil { - errors = append(errors, err) - } - } - } - - return errors -} - -func (cm *Manager) reload(t string, blocks []*api.ConfigBlock) *xmanagement.Error { - cm.logger.Infof("Applying settings for %s", t) - if obj := cm.registry.GetReloadable(t); obj != nil { - // Single object - if len(blocks) > 1 { - err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) - cm.logger.Error(err) - return xmanagement.NewConfigError(err) - } - - var config *reload.ConfigWithMeta - var err error - if len(blocks) == 1 { - config, err = blocks[0].ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return xmanagement.NewConfigError(err) - } - } - - if err := obj.Reload(config); err != nil { - cm.logger.Error(err) - return xmanagement.NewConfigError(err) - } - } else if obj := cm.registry.GetReloadableList(t); obj != nil { - // List - var configs []*reload.ConfigWithMeta - for _, block := range blocks { - config, err := block.ConfigWithMeta() - if err != nil { - cm.logger.Error(err) - return xmanagement.NewConfigError(err) - } - configs = append(configs, config) - } - - if err := obj.Reload(configs); err != nil { - cm.logger.Error(err) - return xmanagement.NewConfigError(err) - } - } - - return nil -} - -func (cm *Manager) toConfigBlocks(cfg common.MapStr) (api.ConfigBlocks, error) { - blocks := map[string][]*api.ConfigBlock{} - - // Extract all registered values beat can respond to - for _, regName := range cm.registry.GetRegisteredNames() { - iBlock, err := cfg.GetValue(regName) - if err != nil { - continue - } - - if mapBlock, ok := iBlock.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &api.ConfigBlock{Raw: mapBlock}) - } else if arrayBlock, ok := iBlock.([]interface{}); ok { - for _, item := range arrayBlock { - if mapBlock, ok := item.(map[string]interface{}); ok { - blocks[regName] = append(blocks[regName], &api.ConfigBlock{Raw: mapBlock}) - } - } - } - } - - // keep the ordering consistent while grouping the items. - keys := make([]string, 0, len(blocks)) - for k := range blocks { - keys = append(keys, k) - } - sort.Strings(keys) - - res := api.ConfigBlocks{} - for _, t := range keys { - b := blocks[t] - res = append(res, api.ConfigBlocksWithType{Type: t, Blocks: b}) - } - - return res, nil -} - -func statusToProtoStatus(status management.Status) proto.StateObserved_Status { - switch status { - case management.Unknown: - // unknown is reported as healthy, as the status is unknown - return proto.StateObserved_HEALTHY - case management.Starting: - return proto.StateObserved_STARTING - case management.Configuring: - return proto.StateObserved_CONFIGURING - case management.Running: - return proto.StateObserved_HEALTHY - case management.Degraded: - return proto.StateObserved_DEGRADED - case management.Failed: - return proto.StateObserved_FAILED - case management.Stopping: - return proto.StateObserved_STOPPING - } - // unknown status, still reported as healthy - return proto.StateObserved_HEALTHY -} diff --git a/x-pack/libbeat/management/fleet/manager_test.go b/x-pack/libbeat/management/fleet/manager_test.go deleted file mode 100644 index 7810886018a3..000000000000 --- a/x-pack/libbeat/management/fleet/manager_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/management" -) - -func TestConfigBlocks(t *testing.T) { - input := ` -filebeat: - inputs: - - type: log - paths: - - /var/log/hello1.log - - /var/log/hello2.log -output: - elasticsearch: - hosts: - - localhost:9200` - - var cfg common.MapStr - uconfig, err := common.NewConfigFrom(input) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - err = uconfig.Unpack(&cfg) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - reg := reload.NewRegistry() - reg.Register("output", &dummyReloadable{}) - reg.Register("filebeat.inputs", &dummyReloadable{}) - - cm := &Manager{ - registry: reg, - } - blocks, err := cm.toConfigBlocks(cfg) - if err != nil { - t.Fatalf("Config blocks unsuccessfully generated: %+v", err) - } - - if len(blocks) != 2 { - t.Fatalf("Expected 2 block have %d: %+v", len(blocks), blocks) - } -} - -func TestStatusToProtoStatus(t *testing.T) { - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Unknown)) - assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(management.Starting)) - assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(management.Configuring)) - assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(management.Running)) - assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(management.Degraded)) - assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(management.Failed)) - assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(management.Stopping)) -} - -type dummyReloadable struct{} - -func (dummyReloadable) Reload(config *reload.ConfigWithMeta) error { - return nil -} diff --git a/x-pack/libbeat/management/fleet/plugin.go b/x-pack/libbeat/management/fleet/plugin.go deleted file mode 100644 index edb12ca78f18..000000000000 --- a/x-pack/libbeat/management/fleet/plugin.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package fleet - -import ( - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/management" - xmanagement "github.com/elastic/beats/v7/x-pack/libbeat/management" -) - -func init() { - management.Register("x-pack-fleet", NewFleetManagerPlugin, feature.Beta) -} - -// NewFleetManagerPlugin creates a plugin function returning factory if configuration matches the criteria -func NewFleetManagerPlugin(config *common.Config) management.FactoryFunc { - c := defaultConfig() - if config.Enabled() { - if err := config.Unpack(&c); err != nil { - return nil - } - - if c.Mode == xmanagement.ModeFleet { - return NewFleetManager - } - } - - return nil -} diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index cfbf72f234cd..36a4f8c0245a 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -5,269 +5,217 @@ package management import ( + "context" "fmt" + "os" + "sort" "sync" - "time" - - "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/gofrs/uuid" - + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/management" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/common/reload" + "github.com/elastic/beats/v7/libbeat/logp" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) -var errEmptyAccessToken = errors.New("access_token is empty, you must reenroll your Beat") - -// ConfigManager handles internal config updates. By retrieving -// new configs from Kibana and applying them to the Beat -type ConfigManager struct { +// Manager handles internal config updates. By retrieving +// new configs from Kibana and applying them to the Beat. +type Manager struct { config *Config - cache *Cache logger *logp.Logger - client api.AuthClienter beatUUID uuid.UUID - done chan struct{} registry *reload.Registry - wg sync.WaitGroup blacklist *ConfigBlacklist - reporter *api.EventReporter - state *State - mux sync.RWMutex + client client.Client + lock sync.Mutex + status lbmanagement.Status + msg string + payload map[string]interface{} + + stopFunc func() } -// NewConfigManager returns a X-Pack Beats Central Management manager -func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { +// NewFleetManager returns a X-Pack Beats Fleet Management manager. +func NewFleetManager(config *common.Config, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { - return nil, errors.Wrap(err, "parsing central management settings") + return nil, errors.Wrap(err, "parsing fleet management settings") } } - return NewConfigManagerWithConfig(c, registry, beatUUID) + return NewFleetManagerWithConfig(c, registry, beatUUID) } -// NewConfigManagerWithConfig returns a X-Pack Beats Central Management manager -func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.Manager, error) { - var client *api.Client - var cache *Cache - var blacklist *ConfigBlacklist - - if c.Enabled { - cfgwarn.Deprecate("8.0.0", "Central Management is no longer under development and has been deprecated. We are working hard to deliver a new and more comprehensive solution and look forward to sharing it with you") - - var err error +// NewFleetManagerWithConfig returns a X-Pack Beats Fleet Management manager. +func NewFleetManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (lbmanagement.Manager, error) { + log := logp.NewLogger(lbmanagement.DebugK) - if err = validateConfig(c); err != nil { - return nil, errors.Wrap(err, "wrong settings for configurations") - } + m := &Manager{ + config: c, + logger: log.Named("fleet"), + beatUUID: beatUUID, + registry: registry, + } + var err error + var blacklist *ConfigBlacklist + var eac client.Client + if c.Enabled { // Initialize configs blacklist blacklist, err = NewConfigBlacklist(c.Blacklist) if err != nil { return nil, errors.Wrap(err, "wrong settings for configurations blacklist") } - // Initialize central management settings cache - cache = &Cache{} - if err := cache.Load(); err != nil { - return nil, errors.Wrap(err, "reading central management internal cache") - } - - // Ignore kibana version to avoid permission errors - c.Kibana.IgnoreVersion = true - - client, err = api.NewClient(c.Kibana) + // Initialize the client + eac, err = client.NewFromReader(os.Stdin, m) if err != nil { - return nil, errors.Wrap(err, "initializing kibana client") + return nil, errors.Wrap(err, "failed to create elastic-agent-client") } } - authClient := &api.AuthClient{Client: client, AccessToken: c.AccessToken, BeatUUID: beatUUID} - log := logp.NewLogger(management.DebugK) - - return &ConfigManager{ - config: c, - cache: cache, - blacklist: blacklist, - logger: log, - client: authClient, - done: make(chan struct{}), - beatUUID: beatUUID, - registry: registry, - reporter: api.NewEventReporter( - log, - authClient, - c.EventsReporter.Period, - c.EventsReporter.MaxBatchSize, - ), - }, nil + m.blacklist = blacklist + m.client = eac + return m, nil } -// Enabled returns true if config management is enabled -func (cm *ConfigManager) Enabled() bool { +// Enabled returns true if config management is enabled. +func (cm *Manager) Enabled() bool { return cm.config.Enabled } -func (cm *ConfigManager) RegisterAction(action client.Action) {} - -func (cm *ConfigManager) UnregisterAction(action client.Action) {} - -func (cm *ConfigManager) SetPayload(map[string]interface{}) {} - // Start the config manager -func (cm *ConfigManager) Start(_ func()) { +func (cm *Manager) Start(stopFunc func()) { if !cm.Enabled() { return } - cfgwarn.Beta("Central management is enabled") - cm.logger.Info("Starting central management service") - cm.reporter.Start() - cm.wg.Add(1) - go cm.worker() + cfgwarn.Beta("Fleet management is enabled") + cm.logger.Info("Starting fleet management service") + + cm.stopFunc = stopFunc + err := cm.client.Start(context.Background()) + if err != nil { + cm.logger.Errorf("failed to start elastic-agent-client: %s", err) + } } // Stop the config manager -func (cm *ConfigManager) Stop() { +func (cm *Manager) Stop() { if !cm.Enabled() { return } - // stop collecting configuration - cm.logger.Info("Stopping central management service") - close(cm.done) - cm.wg.Wait() - - // report last state and stop reporting. - cm.updateState(Stopped) - cm.reporter.Stop() + cm.logger.Info("Stopping fleet management service") + cm.client.Stop() } // CheckRawConfig check settings are correct to start the beat. This method // checks there are no collision between the existing configuration and what -// central management can configure. -func (cm *ConfigManager) CheckRawConfig(cfg *common.Config) error { +// fleet management can configure. +func (cm *Manager) CheckRawConfig(cfg *common.Config) error { // TODO implement this method return nil } // UpdateStatus updates the manager with the current status for the beat. -func (cm *ConfigManager) UpdateStatus(_ management.Status, _ string) { - // do nothing; no longer under development and has been deprecated -} - -func (cm *ConfigManager) worker() { - defer cm.wg.Done() - - // Initial fetch && apply (even if errors happen while fetching) - firstRun := true - period := 0 * time.Second - - cm.updateState(Starting) - - // Start worker loop: fetch + apply + cache new settings - for { - select { - case <-cm.done: - return - case <-time.After(period): - } - - changed := cm.fetch() - if changed || firstRun { - cm.updateState(InProgress) - // configs changed, apply changes - // TODO only reload the blocks that changed - if errs := cm.apply(); !errs.IsEmpty() { - cm.reportErrors(errs) - cm.updateState(Failed) - cm.logger.Errorf("Could not apply the configuration, error: %+v", errs) - } else { - cm.updateState(Running) - } - } - - if changed { - // store new configs (already applied) - cm.logger.Info("Storing new state") - if err := cm.cache.Save(); err != nil { - cm.logger.Errorf("error storing central management state: %s", err) - } - } - - if firstRun { - period = cm.config.Period - firstRun = false - } +func (cm *Manager) UpdateStatus(status lbmanagement.Status, msg string) { + cm.lock.Lock() + defer cm.lock.Unlock() + + if cm.status != status || cm.msg != msg { + cm.status = status + cm.msg = msg + cm.client.Status(statusToProtoStatus(status), msg, nil) + cm.logger.Infof("Status change to %s: %s", status, msg) } } -func (cm *ConfigManager) reportErrors(errs Errors) { - for _, err := range errs { - cm.reporter.AddEvent(err) - } -} +func (cm *Manager) OnConfig(s string) { + cm.UpdateStatus(lbmanagement.Configuring, "Updating configuration") -// fetch configurations from kibana, return true if they changed -func (cm *ConfigManager) fetch() bool { - cm.logger.Debug("Retrieving new configurations from Kibana") - configs, err := cm.client.Configuration() - - if api.IsConfigurationNotFound(err) { - if cm.cache.HasConfig() { - cm.logger.Error("Disabling all running configuration because no configurations were found for this Beat, the endpoint returned a 404 or the beat is not enrolled with central management") - cm.cache.Configs = api.ConfigBlocks{} - } - return true + var configMap common.MapStr + uconfig, err := common.NewConfigFrom(s) + if err != nil { + err = errors.Wrap(err, "config blocks unsuccessfully generated") + cm.logger.Error(err) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) + return } + err = uconfig.Unpack(&configMap) if err != nil { - cm.logger.Errorf("error retrieving new configurations, will use cached ones: %s", err) - return false + err = errors.Wrap(err, "config blocks unsuccessfully generated") + cm.logger.Error(err) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) + return } - equal, err := api.ConfigBlocksEqual(configs, cm.cache.Configs) + blocks, err := cm.toConfigBlocks(configMap) if err != nil { - cm.logger.Errorf("error comparing the configurations, will use cached ones: %s", err) - return false + err = errors.Wrap(err, "failed to parse configuration") + cm.logger.Error(err) + cm.UpdateStatus(lbmanagement.Failed, err.Error()) + return } - if equal { - cm.logger.Debug("configuration didn't change, sleeping") - return false + if errs := cm.apply(blocks); errs != nil { + // `cm.apply` already logs the errors; currently allow beat to run degraded + cm.UpdateStatus(lbmanagement.Failed, errs.Error()) + return } - cm.logger.Info("New configurations retrieved") - cm.cache.Configs = configs + cm.client.Status(proto.StateObserved_HEALTHY, "Running", cm.payload) +} + +func (cm *Manager) RegisterAction(action client.Action) { + cm.client.RegisterAction(action) +} + +func (cm *Manager) UnregisterAction(action client.Action) { + cm.client.UnregisterAction(action) +} + +func (cm *Manager) SetPayload(payload map[string]interface{}) { + cm.lock.Lock() + cm.payload = payload + cm.lock.Unlock() +} + +func (cm *Manager) OnStop() { + if cm.stopFunc != nil { + cm.client.Status(proto.StateObserved_STOPPING, "Stopping", nil) + cm.stopFunc() + } +} - return true +func (cm *Manager) OnError(err error) { + cm.logger.Errorf("elastic-agent-client got error: %s", err) } -func (cm *ConfigManager) apply() Errors { - var errors Errors +func (cm *Manager) apply(blocks ConfigBlocks) error { missing := map[string]bool{} for _, name := range cm.registry.GetRegisteredNames() { missing[name] = true } // Detect unwanted configs from the list - if errs := cm.blacklist.Detect(cm.cache.Configs); !errs.IsEmpty() { - errors = append(errors, errs...) - return errors + if err := cm.blacklist.Detect(blocks); err != nil { + return err } + var errors *multierror.Error // Reload configs - for _, b := range cm.cache.Configs { + for _, b := range blocks { if err := cm.reload(b.Type, b.Blocks); err != nil { - errors = append(errors, err) + errors = multierror.Append(errors, err) } missing[b.Type] = false } @@ -275,23 +223,23 @@ func (cm *ConfigManager) apply() Errors { // Unset missing configs for name := range missing { if missing[name] { - if err := cm.reload(name, []*api.ConfigBlock{}); err != nil { - errors = append(errors, err) + if err := cm.reload(name, []*ConfigBlock{}); err != nil { + errors = multierror.Append(errors, err) } } } - return errors + return errors.ErrorOrNil() } -func (cm *ConfigManager) reload(t string, blocks []*api.ConfigBlock) *Error { +func (cm *Manager) reload(t string, blocks []*ConfigBlock) error { cm.logger.Infof("Applying settings for %s", t) if obj := cm.registry.GetReloadable(t); obj != nil { // Single object if len(blocks) > 1 { err := fmt.Errorf("got an invalid number of configs for %s: %d, expected: 1", t, len(blocks)) cm.logger.Error(err) - return NewConfigError(err) + return err } var config *reload.ConfigWithMeta @@ -300,13 +248,13 @@ func (cm *ConfigManager) reload(t string, blocks []*api.ConfigBlock) *Error { config, err = blocks[0].ConfigWithMeta() if err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } if err := obj.Reload(config); err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } else if obj := cm.registry.GetReloadableList(t); obj != nil { // List @@ -315,31 +263,75 @@ func (cm *ConfigManager) reload(t string, blocks []*api.ConfigBlock) *Error { config, err := block.ConfigWithMeta() if err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } configs = append(configs, config) } if err := obj.Reload(configs); err != nil { cm.logger.Error(err) - return NewConfigError(err) + return err } } return nil } -func (cm *ConfigManager) updateState(state State) { - cm.mux.Lock() - defer cm.mux.Unlock() - cm.state = &state - cm.reporter.AddEvent(&state) - cm.logger.Infof("Updating state to '%s'", state) +func (cm *Manager) toConfigBlocks(cfg common.MapStr) (ConfigBlocks, error) { + blocks := map[string][]*ConfigBlock{} + + // Extract all registered values beat can respond to + for _, regName := range cm.registry.GetRegisteredNames() { + iBlock, err := cfg.GetValue(regName) + if err != nil { + continue + } + + if mapBlock, ok := iBlock.(map[string]interface{}); ok { + blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) + } else if arrayBlock, ok := iBlock.([]interface{}); ok { + for _, item := range arrayBlock { + if mapBlock, ok := item.(map[string]interface{}); ok { + blocks[regName] = append(blocks[regName], &ConfigBlock{Raw: mapBlock}) + } + } + } + } + + // keep the ordering consistent while grouping the items. + keys := make([]string, 0, len(blocks)) + for k := range blocks { + keys = append(keys, k) + } + sort.Strings(keys) + + res := ConfigBlocks{} + for _, t := range keys { + b := blocks[t] + res = append(res, ConfigBlocksWithType{Type: t, Blocks: b}) + } + + return res, nil } -func validateConfig(config *Config) error { - if len(config.AccessToken) == 0 { - return errEmptyAccessToken +func statusToProtoStatus(status lbmanagement.Status) proto.StateObserved_Status { + switch status { + case lbmanagement.Unknown: + // unknown is reported as healthy, as the status is unknown + return proto.StateObserved_HEALTHY + case lbmanagement.Starting: + return proto.StateObserved_STARTING + case lbmanagement.Configuring: + return proto.StateObserved_CONFIGURING + case lbmanagement.Running: + return proto.StateObserved_HEALTHY + case lbmanagement.Degraded: + return proto.StateObserved_DEGRADED + case lbmanagement.Failed: + return proto.StateObserved_FAILED + case lbmanagement.Stopping: + return proto.StateObserved_STOPPING } - return nil + // unknown status, still reported as healthy + return proto.StateObserved_HEALTHY } diff --git a/x-pack/libbeat/management/manager_test.go b/x-pack/libbeat/management/manager_test.go index c783989b7082..113c19efe5ce 100644 --- a/x-pack/libbeat/management/manager_test.go +++ b/x-pack/libbeat/management/manager_test.go @@ -5,468 +5,70 @@ package management import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "sync" "testing" - "time" - "github.com/gofrs/uuid" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/beats/v7/libbeat/paths" - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) -type reloadable struct { - reloaded chan *reload.ConfigWithMeta -} - -func (r *reloadable) Reload(c *reload.ConfigWithMeta) error { - r.reloaded <- c - return nil -} - -func TestConfigManager(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.block", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []string{ - // Initial load - `{"success": true, "list":[{"type":"test.block","config":{"module":"apache2"}}]}`, - - // No change, no reload - `{"success": true, "list":[{"type":"test.block","config":{"module":"apache2"}}]}`, - - // Changed, reload - `{"success": true, "list":[{"type":"test.block","config":{"module":"system"}}]}`, - } - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, responses[i]) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - config2 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "system", - }), - }, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func TestRemoveItems(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []string{ - // Initial load - `{"success": true, "list":[{"type":"test.blocks","config":{"module":"apache2"}}]}`, - - // Return no blocks - `{"success": true, "list":[]}`, - } - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, responses[i]) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - // Get a nil config, even if the block is not part of the payload - config2 := <-reloadable.reloaded - var nilConfig *reload.ConfigWithMeta - assert.Equal(t, nilConfig, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func responseText(s string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, s) - } -} - -func TestUnEnroll(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() - if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []http.HandlerFunc{ // Initial load - responseText(`{"success": true, "list":[{"type":"test.blocks","config":{"module":"apache2"}}]}`), - http.NotFound, - } - - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - responses[i](w, r) - i++ - })) - - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) +func TestConfigBlocks(t *testing.T) { + input := ` +filebeat: + inputs: + - type: log + paths: + - /var/log/hello1.log + - /var/log/hello2.log +output: + elasticsearch: + hosts: + - localhost:9200` + + var cfg common.MapStr + uconfig, err := common.NewConfigFrom(input) if err != nil { - t.Fatal(err) + t.Fatalf("Config blocks unsuccessfully generated: %+v", err) } - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - } - - manager, err := NewConfigManagerWithConfig(config, registry, id) - if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Equal(t, &reload.ConfigWithMeta{ - Config: common.MustNewConfigFrom(map[string]interface{}{ - "module": "apache2", - }), - }, config1) - - // Get a nil config, even if the block is not part of the payload - config2 := <-reloadable.reloaded - var nilConfig *reload.ConfigWithMeta - assert.Equal(t, nilConfig, config2) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{&Starting, &InProgress, &Running, &InProgress, &Running, &Stopped} - assertEvents(t, events, reporter) -} - -func TestBadConfig(t *testing.T) { - registry := reload.NewRegistry() - id, err := uuid.NewV4() + err = uconfig.Unpack(&cfg) if err != nil { - t.Fatalf("error while generating id: %v", err) - } - accessToken := "footoken" - reloadable := reloadable{ - reloaded: make(chan *reload.ConfigWithMeta, 1), - } - registry.MustRegister("test.blocks", &reloadable) - - mux := http.NewServeMux() - i := 0 - responses := []http.HandlerFunc{ // Initial load - responseText(`{"success": true, "list":[{"type":"output","config":{"_sub_type": "console", "path": "/tmp/bad"}}]}`), - // will not resend new events - responseText(`{"success": true, "list":[{"type":"output","config":{"_sub_type": "console", "path": "/tmp/bad"}}]}`), - // recover on call - http.NotFound, + t.Fatalf("Config blocks unsuccessfully generated: %+v", err) } - mux.Handle(fmt.Sprintf("/api/beats/agent/%s/configuration", id), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - responses[i](w, r) - i++ - })) + reg := reload.NewRegistry() + reg.Register("output", &dummyReloadable{}) + reg.Register("filebeat.inputs", &dummyReloadable{}) - reporter := addEventsReporterHandle(mux, id) - server := httptest.NewServer(mux) - - c, err := api.ConfigFromURL(server.URL, common.NewConfig()) - if err != nil { - t.Fatal(err) - } - - config := &Config{ - Enabled: true, - Mode: ModeCentralManagement, - Period: 100 * time.Millisecond, - Kibana: c, - AccessToken: accessToken, - EventsReporter: EventReporterConfig{ - Period: 50 * time.Millisecond, - MaxBatchSize: 1, - }, - Blacklist: ConfigBlacklistSettings{ - Patterns: map[string]string{ - "output": "console|file", - }, - }, + cm := &Manager{ + registry: reg, } - - manager, err := NewConfigManagerWithConfig(config, registry, id) + blocks, err := cm.toConfigBlocks(cfg) if err != nil { - t.Fatal(err) - } - - manager.Start(func() {}) - - // On first reload we will get apache2 module - config1 := <-reloadable.reloaded - assert.Nil(t, config1) - - // Cleanup - manager.Stop() - os.Remove(paths.Resolve(paths.Data, "management.yml")) - - events := []api.Event{ - &Starting, - &InProgress, - &Error{Type: ConfigError, Err: errors.New("Config for 'output' is blacklisted")}, - &Failed, - &InProgress, // recovering on NotFound, to get out of the blocking. - &Running, - &Stopped, - } - assertEvents(t, events, reporter) -} - -type testEventRequest struct { - EventType api.EventType - Event api.Event -} - -func (er *testEventRequest) UnmarshalJSON(b []byte) error { - resp := struct { - EventType api.EventType `json:"type"` - Event json.RawMessage `json:"event"` - }{} - - if err := json.Unmarshal(b, &resp); err != nil { - return err - } - - switch resp.EventType { - case ErrorEvent: - event := &Error{} - if err := json.Unmarshal(resp.Event, event); err != nil { - return err - } - *er = testEventRequest{EventType: resp.EventType, Event: event} - return nil - case StateEvent: - event := State("") - if err := json.Unmarshal(resp.Event, &event); err != nil { - return err - } - *er = testEventRequest{EventType: resp.EventType, Event: &event} - return nil + t.Fatalf("Config blocks unsuccessfully generated: %+v", err) } - return fmt.Errorf("unknown event type of '%s'", resp.EventType) -} - -// collect in the background any events generated from the managers. -type collectEventRequest struct { - sync.Mutex - requests []testEventRequest -} -func (r *collectEventRequest) Requests() []testEventRequest { - r.Lock() - defer r.Unlock() - return r.requests -} - -func (r *collectEventRequest) Add(requests ...testEventRequest) { - r.Lock() - defer r.Unlock() - r.requests = append(r.requests, requests...) -} - -func addEventsReporterHandle(mux *http.ServeMux, uuid uuid.UUID) *collectEventRequest { - reporter := &collectEventRequest{} - path := "/api/beats/" + uuid.String() + "/events" - fn := func(w http.ResponseWriter, r *http.Request) { - var requests []testEventRequest - decoder := json.NewDecoder(r.Body) - if err := decoder.Decode(&requests); err != nil { - http.Error(w, "could not decode JSON", 500) - } - - reporter.Add(requests...) - resp := api.EventAPIResponse{Response: make([]api.EventResponse, len(requests))} - - for i := 0; i < len(requests); i++ { - resp.Response[i] = api.EventResponse{BaseResponse: api.BaseResponse{Success: true}} - } - - json.NewEncoder(w).Encode(resp) + if len(blocks) != 2 { + t.Fatalf("Expected 2 block have %d: %+v", len(blocks), blocks) } - mux.Handle(path, http.HandlerFunc(fn)) - return reporter } -func assertEvents(t *testing.T, events []api.Event, reporter *collectEventRequest) { - requests := reporter.Requests() - if !assert.Equal(t, len(events), len(requests)) { - return - } - - for i := 0; i < len(events); i++ { - switch v := requests[i].Event.(type) { - case *State: - assert.Equal(t, events[i], requests[i].Event) - case *Error: - comparable := events[i].(*Error) - assert.Error(t, comparable.Err, v.Err) - default: - t.Fatalf("cannot assert unknown type: %T", requests[i].Event) - } - } +func TestStatusToProtoStatus(t *testing.T) { + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Unknown)) + assert.Equal(t, proto.StateObserved_STARTING, statusToProtoStatus(lbmanagement.Starting)) + assert.Equal(t, proto.StateObserved_CONFIGURING, statusToProtoStatus(lbmanagement.Configuring)) + assert.Equal(t, proto.StateObserved_HEALTHY, statusToProtoStatus(lbmanagement.Running)) + assert.Equal(t, proto.StateObserved_DEGRADED, statusToProtoStatus(lbmanagement.Degraded)) + assert.Equal(t, proto.StateObserved_FAILED, statusToProtoStatus(lbmanagement.Failed)) + assert.Equal(t, proto.StateObserved_STOPPING, statusToProtoStatus(lbmanagement.Stopping)) } -func TestConfigValidate(t *testing.T) { - tests := map[string]struct { - config *common.Config - err bool - }{ - "missing access_token": { - config: common.MustNewConfigFrom(map[string]interface{}{}), - err: true, - }, - "access_token is present": { - config: common.MustNewConfigFrom(map[string]interface{}{"access_token": "abc1234"}), - err: false, - }, - } +type dummyReloadable struct{} - for name, test := range tests { - t.Run(name, func(t *testing.T) { - c := defaultConfig() - err := test.config.Unpack(c) - if assert.NoError(t, err) { - return - } - - err = validateConfig(c) - if test.err { - assert.Error(t, err) - return - } - assert.NoError(t, err) - }) - } +func (dummyReloadable) Reload(config *reload.ConfigWithMeta) error { + return nil } diff --git a/x-pack/libbeat/management/plugin.go b/x-pack/libbeat/management/plugin.go index b1e7b7d13b94..34560513f93b 100644 --- a/x-pack/libbeat/management/plugin.go +++ b/x-pack/libbeat/management/plugin.go @@ -7,24 +7,21 @@ package management import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/management" + lbmanagement "github.com/elastic/beats/v7/libbeat/management" ) func init() { - management.Register("x-pack", NewManagerPlugin, feature.Beta) + lbmanagement.Register("x-pack-fleet", NewFleetManagerPlugin, feature.Beta) } -// NewManagerPlugin creates a plugin function returning factory if configuration matches the criteria -func NewManagerPlugin(config *common.Config) management.FactoryFunc { +// NewFleetManagerPlugin creates a plugin function returning factory if configuration matches the criteria +func NewFleetManagerPlugin(config *common.Config) lbmanagement.FactoryFunc { c := defaultConfig() if config.Enabled() { if err := config.Unpack(&c); err != nil { return nil } - - if c.Mode == ModeCentralManagement { - return NewConfigManager - } + return NewFleetManager } return nil diff --git a/x-pack/libbeat/management/state.go b/x-pack/libbeat/management/state.go deleted file mode 100644 index 00fdc7c169b1..000000000000 --- a/x-pack/libbeat/management/state.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "fmt" - - "github.com/elastic/beats/v7/x-pack/libbeat/management/api" -) - -// StateEvent is a state change notification. -var StateEvent = api.EventType("STATE") - -var ( - // Starting is when the Manager is created and no config are currently active. - Starting = State("STARTING") - // InProgress we have received a new config from the Remote endpoint and we are trying to apply it. - InProgress = State("IN_PROGRESS") - // Running is set when all the config are successfully applied. - Running = State("RUNNING") - // Failed is set if an unpack failed, a a blacklisted option is set or when a reload fails. - Failed = State("FAILED") - // Stopped is set when CM is shutting down, on close the event reported will flush any pending states. - Stopped = State("STOPPED") -) - -var translateState = map[string]State{ - "STARTING": Starting, - "IN_PROGRESS": InProgress, - "RUNNING": Running, - "FAILED": Failed, - "STOPPED": Stopped, -} - -// State represents the internal State of the CM Manager, it does not yet represent -// the full status of beats, because if the manager is marked as Failed it is possible that -// Beat is in fact partially working. A failed state represents an error while unpacking the config -// or when a module failed to reload. -type State string - -// MarshalJSON marshals a status into a valid JSON document. -func (s *State) MarshalJSON() ([]byte, error) { - res := struct { - Type string `json:"type"` - Message string `json:"message"` - }{ - Type: string(*s), - Message: fmt.Sprintf("State change: %s", *s), - } - return json.Marshal(&res) -} - -// EventType returns the type of event. -func (s *State) EventType() api.EventType { - return StateEvent -} - -// UnmarshalJSON unmarshals the State. -func (s *State) UnmarshalJSON(b []byte) error { - raw := struct { - Type string `json:"type"` - }{} - - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - - v, ok := translateState[raw.Type] - if !ok { - return fmt.Errorf("unknown state %s", raw.Type) - } - - *s = v - return nil -} - -func (s *State) String() string { - return string(*s) -} diff --git a/x-pack/libbeat/management/state_test.go b/x-pack/libbeat/management/state_test.go deleted file mode 100644 index 2f0ab7df866b..000000000000 --- a/x-pack/libbeat/management/state_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package management - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSerializationOfState(t *testing.T) { - t.Run("serialize ok", func(t *testing.T) { - e := &Starting - - b, err := json.Marshal(&e) - if assert.NoError(t, err) { - return - } - - resp := &struct { - Message string `json:"message"` - Type string `json:"type"` - }{} - - err = json.Unmarshal(b, resp) - if assert.NoError(t, err) { - return - } - - assert.Equal(t, e.String(), resp.Type) - assert.NotEmpty(t, resp.Message) - }) - t.Run("ensure that json general fields are present", ensureJSONhasGeneralfield(t, &Starting)) -} - -// Ensure that all events have a Message key that can by used by the GUI. -func ensureJSONhasGeneralfield(t *testing.T, obj json.Marshaler) func(*testing.T) { - return func(t *testing.T) { - serialized, err := json.Marshal(obj) - if !assert.NoError(t, err) { - return - } - - message := struct { - Message string `json:"message"` - }{} - - err = json.Unmarshal(serialized, &message) - - if !assert.NoError(t, err) { - return - } - assert.NotEmpty(t, message) - } -} diff --git a/x-pack/libbeat/tests/system/base.py b/x-pack/libbeat/tests/system/base.py deleted file mode 100644 index ac2fdb84858f..000000000000 --- a/x-pack/libbeat/tests/system/base.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys -import os -from beat.beat import TestCase - - -class BaseTest(TestCase): - - @classmethod - def setUpClass(self): - self.beat_name = "mockbeat" - self.beat_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../")) - self.test_binary = self.beat_path + "/libbeat.test" - super(BaseTest, self).setUpClass() diff --git a/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 b/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 deleted file mode 100644 index 134545b9970c..000000000000 --- a/x-pack/libbeat/tests/system/config/mockbeat.yml.j2 +++ /dev/null @@ -1,93 +0,0 @@ -############################# Mockbeat ###################################### -mockbeat: -############################# General ############################################ - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -name: - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. -tags: [ -{%- if agent_tags -%} - {%- for tag in agent_tags -%} - "{{ tag }}" - {%- if not loop.last %}, {% endif -%} - {%- endfor -%} -{%- endif -%}] - -#================================ Queue ===================================== - -queue.mem: - events: 4096 - flush.min_events: 8 - flush.timeout: 0.1s - -############################# Output ############################################ - -# Configure what outputs to use when sending the data collected by mockbeat. -# Multiple outputs may NOT be enabled. -output: - {% if console -%} - console: - {% for k, v in console.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {%- endif %} - - {% if elasticsearch -%} - elasticsearch: - {% for k, v in elasticsearch.items() -%} - {{ k }}: {{ v }} - {% endfor -%} - {%- endif %} - - # Redis as output - # Options: - # host, port: where Redis is listening on - #redis: - # host: localhost - # port: 6379 - - # File as output - # Options - # path: where to save the files - # filename: name of the files - # rotate_every_kb: maximum size of the files in path - # number of files: maximum number of files in path - {% if not (console or elasticsearch) -%} - file: - path: {{ output_file_path|default(beat.working_dir + "/output") }} - filename: "{{ output_file_filename|default("mockbeat") }}" - rotate_every_kb: 1000 - #number_of_files: 7 - {%- endif %} - -#==================== Elasticsearch template setting ========================== -setup.template: - fields: {{ fields|default("fields.yml") }} - settings: - index.number_of_shards: 1 - index.codec: best_compression - name: {{ es_template_name }} - pattern: {{ es_template_pattern }} - overwrite: {{ template_overwrite|default("false") }} - json: - enabled: {{ template_json_enabled|default("false") }} - path: {{ template_json_path }} - name: {{ template_json_name }} - -#================================ Logging ===================================== - -{% if metrics_period -%} -logging.metrics.period: {{ metrics_period }} -{%- endif %} - -{% if keystore_path %} -#================================ keystore ===================================== -keystore.path: {{keystore_path}} -{% endif %} - -# vim: set ft=jinja: diff --git a/x-pack/libbeat/tests/system/requirements.txt b/x-pack/libbeat/tests/system/requirements.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/x-pack/libbeat/tests/system/test_management.py b/x-pack/libbeat/tests/system/test_management.py deleted file mode 100644 index 35a4f9949f9f..000000000000 --- a/x-pack/libbeat/tests/system/test_management.py +++ /dev/null @@ -1,291 +0,0 @@ -import sys -import os -import glob -import json -import requests -import string -import random -import unittest -import time -from elasticsearch import Elasticsearch -from os import path - - -from base import BaseTest - - -# Disable because waiting artifacts from https://github.com/elastic/kibana/pull/31660 -INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) -# INTEGRATION_TESTS = False -TIMEOUT = 2 * 60 - - -class TestManagement(BaseTest): - - def setUp(self): - super(TestManagement, self).setUp() - # NOTES: Theses options are linked to the specific of the docker compose environment for - # CM. - self.es_host = os.getenv('ES_HOST', 'localhost') + ":" + os.getenv('ES_POST', '9200') - self.es_user = "myelastic" - self.es_pass = "changeme" - self.es = Elasticsearch([self.get_elasticsearch_url()], verify_certs=True) - self.keystore_path = self.working_dir + "/data/keystore" - - if path.exists(self.keystore_path): - os.Remove(self.keystore_path) - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_enroll(self): - """ - Enroll the beat in Kibana Central Management - """ - - assert len(glob.glob(os.path.join(self.working_dir, "mockbeat.yml.*.bak"))) == 0 - - # We don't care about this as it will be replaced by enrollment - # process: - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - - config_content = open(config_path, 'r').read() - - exit_code = self.enroll(self.es_user, self.es_pass) - - assert exit_code == 0 - - assert self.log_contains("Enrolled and ready to retrieve settings") - - # Enroll creates a keystore (to store access token) - assert os.path.isfile(os.path.join( - self.working_dir, "data/keystore")) - - # New settings file is in place now - new_content = open(config_path, 'r').read() - assert config_content != new_content - - # Settings backup has been created - backup_file = glob.glob(os.path.join(self.working_dir, "mockbeat.yml.*.bak"))[0] - assert os.path.isfile(backup_file) - backup_content = open(backup_file).read() - assert config_content == backup_content - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_enroll_bad_pw(self): - """ - Try to enroll the beat in Kibana Central Management with a bad password - """ - # We don't care about this as it will be replaced by enrollment - # process: - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - - config_content = open(config_path, 'r').read() - - exit_code = self.enroll("not", 'wrong password') - - assert exit_code == 1 - - # Keystore wasn't created - assert not os.path.isfile(os.path.join( - self.working_dir, "data/keystore")) - - # Settings hasn't changed - new_content = open(config_path, 'r').read() - assert config_content == new_content - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_fetch_configs(self): - """ - Config is retrieved from Central Management and updates are applied - """ - # Enroll the beat - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - exit_code = self.enroll(self.es_user, self.es_pass) - assert exit_code == 0 - - index = self.random_index() - # Configure an output - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index, - }, - "id": "myconfig", - } - ]) - - # Start beat - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - # Wait for beat to apply new conf - self.wait_log_contains("Applying settings for output") - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - - self.wait_documents(index, 1) - - index2 = self.random_index() - - # Update output configuration - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index2, - }, - "id": "myconfig", - } - ]) - self.wait_log_contains("Applying settings for output") - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index2, 1) - - proc.check_kill_and_wait() - - @unittest.skipIf(not INTEGRATION_TESTS, - "integration tests are disabled, run with INTEGRATION_TESTS=1 to enable them.") - def test_configs_cache(self): - """ - Config cache is used if Kibana is not available - """ - # Enroll the beat - config_path = os.path.join(self.working_dir, "mockbeat.yml") - self.render_config_template("mockbeat", config_path, keystore_path=self.keystore_path) - exit_code = self.enroll(self.es_user, self.es_pass) - assert exit_code == 0 - - index = self.random_index() - - # Update output configuration - self.create_and_assing_tag([ - { - "type": "output", - "config": { - "_sub_type": "elasticsearch", - "hosts": [self.es_host], - "username": self.es_user, - "password": self.es_pass, - "index": index, - } - } - ]) - - # Start beat - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index, 1) - proc.check_kill_and_wait() - - # Remove the index - self.es.indices.delete(index) - - # Cache should exists already, start with wrong kibana settings: - proc = self.start_beat(extra_args=[ - "-E", "management.period=1s", - "-E", "management.kibana.host=wronghost", - "-E", "management.kibana.timeout=0.5s", - "-E", "keystore.path=%s" % self.keystore_path, - ]) - - self.wait_until(lambda: self.log_contains("PublishEvents: "), max_timeout=TIMEOUT) - self.wait_documents(index, 1) - proc.check_kill_and_wait() - - def enroll(self, user, password): - return self.run_beat( - extra_args=["enroll", self.get_kibana_url(), - "--password", "env:PASS", "--username", user, "--force"], - logging_args=["-v", "-d", "*"], - env={ - 'PASS': password, - }) - - def check_kibana_status(self): - headers = { - "kbn-xsrf": "1" - } - - # Create tag - url = self.get_kibana_url() + "/api/status" - - r = requests.get(url, headers=headers, - auth=(self.es_user, self.es_pass)) - - def create_and_assing_tag(self, blocks): - tag_name = "test%d" % int(time.time() * 1000) - headers = { - "kbn-xsrf": "1" - } - - # Create tag - url = self.get_kibana_url() + "/api/beats/tag/" + tag_name - data = { - "color": "#DD0A73", - "name": tag_name, - } - - r = requests.put(url, json=data, headers=headers, - auth=(self.es_user, self.es_pass)) - assert r.status_code in (200, 201) - - # Create blocks - url = self.get_kibana_url() + "/api/beats/configurations" - for b in blocks: - b["tag"] = tag_name - - r = requests.put(url, json=blocks, headers=headers, - auth=(self.es_user, self.es_pass)) - assert r.status_code in (200, 201) - - # Retrieve beat ID - meta = json.loads( - open(os.path.join(self.working_dir, 'data', 'meta.json'), 'r').read()) - - # Assign tag to beat - data = {"assignments": [{"beatId": meta["uuid"], "tag": tag_name}]} - url = self.get_kibana_url() + "/api/beats/agents_tags/assignments" - r = requests.post(url, json=data, headers=headers, - auth=(self.es_user, self.es_pass)) - - assert r.status_code == 200 - - def get_elasticsearch_url(self): - return 'http://' + self.es_user + ":" + self.es_pass + '@' + \ - os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '5601') - - def get_kibana_url(self): - return 'http://' + os.getenv('KIBANA_HOST', 'kibana') + ':' + os.getenv('KIBANA_PORT', '5601') - - def random_index(self): - return ''.join(random.choice(string.ascii_lowercase) for i in range(10)) - - def check_document_count(self, index, count): - try: - self.es.indices.refresh(index=index) - return self.es.search(index=index, body={"query": {"match_all": {}}})['hits']['total']['value'] >= count - except BaseException: - return False - - def wait_documents(self, index, count): - self.wait_until(lambda: self.check_document_count(index, count), max_timeout=TIMEOUT, poll_interval=1) diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 242657049beb..f5a3ce207f20 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -16,9 +16,9 @@ import ( "github.com/elastic/beats/v7/metricbeat/beater" mbcmd "github.com/elastic/beats/v7/metricbeat/cmd" "github.com/elastic/beats/v7/metricbeat/cmd/test" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register the includes. + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" _ "github.com/elastic/beats/v7/x-pack/metricbeat/include" // Import OSS modules. @@ -57,5 +57,4 @@ func init() { RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), settings) RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", beater.DefaultTestModulesCreator())) - xpackcmd.AddXPack(RootCmd, Name) } diff --git a/x-pack/osquerybeat/cmd/root.go b/x-pack/osquerybeat/cmd/root.go index adaa112619f3..c4548bac2006 100644 --- a/x-pack/osquerybeat/cmd/root.go +++ b/x-pack/osquerybeat/cmd/root.go @@ -9,7 +9,8 @@ import ( cmd "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of this beat @@ -24,6 +25,5 @@ func Osquerybeat() *cmd.BeatsRootCmd { } command := cmd.GenRootCmdWithSettings(beater.New, settings) - xpackcmd.AddXPack(command, Name) return command } diff --git a/x-pack/packetbeat/cmd/root.go b/x-pack/packetbeat/cmd/root.go index e5dd5bb7918c..c7c15b058be7 100644 --- a/x-pack/packetbeat/cmd/root.go +++ b/x-pack/packetbeat/cmd/root.go @@ -7,7 +7,8 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" packetbeatCmd "github.com/elastic/beats/v7/packetbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" + + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" ) // Name of this beat. @@ -20,5 +21,4 @@ func init() { settings := packetbeatCmd.PacketbeatSettings() settings.ElasticLicensed = true RootCmd = packetbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, packetbeatCmd.Name) } diff --git a/x-pack/winlogbeat/cmd/root.go b/x-pack/winlogbeat/cmd/root.go index bc6cd4b0afde..ae56c4de83fe 100644 --- a/x-pack/winlogbeat/cmd/root.go +++ b/x-pack/winlogbeat/cmd/root.go @@ -7,9 +7,9 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" winlogbeatCmd "github.com/elastic/beats/v7/winlogbeat/cmd" - xpackcmd "github.com/elastic/beats/v7/x-pack/libbeat/cmd" // Register fields. + _ "github.com/elastic/beats/v7/x-pack/libbeat/include" _ "github.com/elastic/beats/v7/x-pack/winlogbeat/include" ) @@ -23,5 +23,4 @@ func init() { settings := winlogbeatCmd.WinlogbeatSettings() settings.ElasticLicensed = true RootCmd = winlogbeatCmd.Initialize(settings) - xpackcmd.AddXPack(RootCmd, winlogbeatCmd.Name) }