diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8e8e99a9e12..734033bea89 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -141,6 +141,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Functionbeat* +- Support for Google Cloud Functions have been removed, as it has been in Beta for a long time and been broken +for a few releases. Please use other tools provided by Elastic to fetch data from GCP (e.g. Filebeat). + ==== Bugfixes diff --git a/NOTICE.txt b/NOTICE.txt index 41518c53964..5dfebf5dbec 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -645,218 +645,6 @@ Contents of probable licence file $GOMODCACHE/cloud.google.com/go/pubsub@v1.3.1/ limitations under the License. --------------------------------------------------------------------------------- -Dependency : cloud.google.com/go/storage -Version: v1.10.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/cloud.google.com/go/storage@v1.10.0/LICENSE: - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : code.cloudfoundry.org/go-loggregator Version: v7.4.0+incompatible @@ -19348,6 +19136,218 @@ Contents of probable licence file $GOMODCACHE/k8s.io/client-go@v0.21.1/LICENSE: Indirect dependencies +-------------------------------------------------------------------------------- +Dependency : cloud.google.com/go/storage +Version: v1.10.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/cloud.google.com/go/storage@v1.10.0/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : code.cloudfoundry.org/go-diodes Version: v0.0.0-20190809170250-f77fb823c7ee diff --git a/go.mod b/go.mod index 20cd451a630..439b9d09cf2 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( cloud.google.com/go v0.83.0 cloud.google.com/go/bigquery v1.8.0 cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.10.0 code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee // indirect code.cloudfoundry.org/go-loggregator v7.4.0+incompatible code.cloudfoundry.org/rfc5424 v0.0.0-20180905210152-236a6d29298a // indirect diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 325dca6daf3..ed7b9aa407c 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -31,7 +31,7 @@ stages: parameters: - "armTest" unitTest: - mage: "mage build unitTest && GO_VERSION=1.13.1 mage testGCPFunctions" + mage: "mage build unitTest" stage: mandatory goIntegTest: mage: "mage goIntegTest" diff --git a/x-pack/functionbeat/Makefile b/x-pack/functionbeat/Makefile index 60d069da395..be4e2ceaeb3 100644 --- a/x-pack/functionbeat/Makefile +++ b/x-pack/functionbeat/Makefile @@ -8,7 +8,3 @@ ES_BEATS?=../../ # Includes # include $(ES_BEATS)/dev-tools/make/mage.mk - -.PHONY: test-gcp-functions -test-gcp-functions: mage - mage testGCPFunctions diff --git a/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl index 64d00a2fc1c..c5b5b0524fa 100644 --- a/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl +++ b/x-pack/functionbeat/_meta/config/beat.reference.yml.tmpl @@ -295,111 +295,3 @@ functionbeat.provider.aws.functions: # Set to true to publish fields with null values in events. #keep_null: false - -# Configure functions to run on Google Cloud Platform, currently we assume that the credentials -# are present in the environment to correctly create the function when using the CLI. -# -# Configure which region your project is located in. -functionbeat.provider.gcp.location_id: "europe-west2" -# Configure which Google Cloud project to deploy your functions. -functionbeat.provider.gcp.project_id: "my-project-123456" -# Configure the Google Cloud Storage we should upload the function artifact. -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" - -functionbeat.provider.gcp.functions: - # Define the list of function availables, each function required to have a unique name. - # Create a function that accepts events coming from Google Pub/Sub. - - name: pubsub - enabled: false - type: pubsub - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Pub/Sub" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - trigger: - resource: "projects/_/pubsub/myPubSub" - #service: "pubsub.googleapis.com" - - # Set to true to publish fields with null values in events. - #keep_null: false - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" - - # Create a function that accepts events coming from Google Cloud Storage. - - name: storage - enabled: false - type: storage - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Cloud Storage" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - trigger: - resource: "projects/my-project/buckets/my-storage" - #event_type: "google.storage.object.finalize" - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - # Set to true to publish fields with null values in events. - #keep_null: false - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" diff --git a/x-pack/functionbeat/_meta/config/beat.yml.tmpl b/x-pack/functionbeat/_meta/config/beat.yml.tmpl index 00caf63d94c..48c0e6ae40b 100644 --- a/x-pack/functionbeat/_meta/config/beat.yml.tmpl +++ b/x-pack/functionbeat/_meta/config/beat.yml.tmpl @@ -251,104 +251,3 @@ functionbeat.provider.aws.functions: # Default is 1. #parallelization_factor: 1 -# Configure functions to run on Google Cloud Platform, currently we assume that the credentials -# are present in the environment to correctly create the function when using the CLI. -# -# Configure which region your project is located in. -functionbeat.provider.gcp.location_id: "europe-west2" -# Configure which Google Cloud project to deploy your functions. -functionbeat.provider.gcp.project_id: "my-project-123456" -# Configure the Google Cloud Storage we should upload the function artifact. -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" - -functionbeat.provider.gcp.functions: - # Define the list of function availables, each function required to have a unique name. - # Create a function that accepts events coming from Google Pub/Sub. - - name: pubsub - enabled: false - type: pubsub - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Pub/Sub" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - trigger: - resource: "projects/_/pubsub/myPubSub" - #service: "pubsub.googleapis.com" - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" - - # Create a function that accepts events coming from Google Cloud Storage. - - name: storage - enabled: false - type: storage - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Cloud Storage" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - trigger: - resource: "projects/my-project/buckets/my-storage" - #event_type: "google.storage.object.finalize" - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" -#==================== Elasticsearch template setting ========================== - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false diff --git a/x-pack/functionbeat/dev-tools/packaging/packages.yml b/x-pack/functionbeat/dev-tools/packaging/packages.yml index e6c3346bf1a..480b31820b5 100644 --- a/x-pack/functionbeat/dev-tools/packaging/packages.yml +++ b/x-pack/functionbeat/dev-tools/packaging/packages.yml @@ -65,18 +65,6 @@ shared: pkg/functionbeat-aws: source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64' mode: 0755 - pkg/pubsub/vendor: - source: 'provider/gcp/build/pubsub/vendor' - mode: 0644 - pkg/storage/vendor: - source: 'provider/gcp/build/storage/vendor' - mode: 0644 - pkg/pubsub/pubsub.go: - source: 'provider/gcp/pubsub/pubsub.go' - mode: 0655 - pkg/storage/storage.go: - source: 'provider/gcp/storage/storage.go' - mode: 0655 # specs is a list of named packaging "flavors". specs: diff --git a/x-pack/functionbeat/docs/config-options-gcp.asciidoc b/x-pack/functionbeat/docs/config-options-gcp.asciidoc deleted file mode 100644 index 6ee6a1f21df..00000000000 --- a/x-pack/functionbeat/docs/config-options-gcp.asciidoc +++ /dev/null @@ -1,198 +0,0 @@ -[id="configuration-{beatname_lc}-gcp-options"] -[role="xpack"] -== Configure Google Functions - -++++ -Google functions -++++ - -beta[] - -{beatname_uc} runs as a Google Function on Google Cloud Platform (GCP). - -Before deploying {beatname_uc}, you need to configure one or more functions and -specify details about the services that will trigger the functions. - -You configure the functions in the the +{beatname_lc}.yml+ configuration file. -When you're done, you can <> -to your serverless environment. - -The following example configures two functions: `pubsub` and `storage`. The -`pubsub` function collects log events from https://cloud.google.com/pubsub/[Google -Pub/Sub]. The `storage` function collects log events from -https://cloud.google.com/storage/[Google Cloud Storage]. Both functions in the -example forward the events to {es}. - -["source","sh",subs="attributes"] ----- -functionbeat.provider.gcp.location_id: "europe-west2" -functionbeat.provider.gcp.project_id: "my-project-123456" -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" -functionbeat.provider.gcp.functions: - - name: pubsub - enabled: true - type: pubsub - description: "Google Cloud Function for Pub/Sub" - trigger: - resource: "projects/_/pubsub/myPubSub" - #service: "pubsub.googleapis.com" - - name: storage - enabled: true - type: storage - description: "Google Cloud Function for Cloud Storage" - trigger: - resource: "projects/my-project/buckets/my-storage" - event_type: "google.storage.object.finalize" - -cloud.id: "MyESDeployment:SomeLongString==" -cloud.auth: "elastic:mypassword" ----- - -[id="{beatname_lc}-gcp-options"] -[float] -=== Configuration options -Specify the following options to configure the functions -that you want to deploy to Google Cloud Platform (GCP). - -TIP: If you change the configuration after deploying the function, use -the <> to update your deployment. - -[float] -[id="{beatname_lc}-gcp-location_id"] -==== `provider.gcp.location_id` - -The region where your GCP project is located. - -[float] -[id="{beatname_lc}-gcp-project_id"] -==== `provider.gcp.project_id` - -The ID of the GCP project where the function artifacts will be deployed. See the -https://cloud.google.com/about/locations/[Google Cloud Function documentation] -to verify that Cloud Functions are supported in the region you specify. - -[float] -[id="{beatname_lc}-gcp-storage_name"] -==== `provider.gcp.storage_name` - -The name of the Google Cloud storage bucket where the function artifacts will be -deployed. If the bucket doesn't exist, it will be created, if you have the -correct project permissions (`storage.objects.create`). - -[float] -[id="{beatname_lc}-gcp-functions"] -==== `functionbeat.provider.gcp.functions` -A list of functions that are available for deployment. - -[float] -[id="{beatname_lc}-gcp-name"] -===== `name` - -A unique name for the Google function. - -[float] -[id="{beatname_lc}-gcp--type"] -===== `type` - -The type of GCP service to monitor. For this release, the supported types -are: - -[horizontal] -`pubsub`:: Collect log events from Google Pub/Sub. -`storage`:: Collect log events from Google Cloud storage buckets. - -[float] -[id="{beatname_lc}-gcp-description"] -===== `description` - -A description of the function. This description is useful when you are running -multiple functions and need more context about how each function is used. - -[float] -[id="{beatname_lc}-gcp-memory-size"] -==== `memory_size` - -The maximum amount of memory to allocate for this function. -The default is `256MB`. - -[float] -[id="{beatname_lc}-gcp-timeout"] -==== `timeout` - -The execution timeout in seconds. If the function does not finish in time, -it is considered failed and terminated. The default is `60s`. Increase this -value if you see timeout messages is the Google Stackdriver logs. - -[float] -[id="{beatname_lc}-gcp-service_account_email"] -==== `service_account_email` - -The email of the service account that the function will assume as its identity. -The default is {projectid}@appspot.gserviceaccount.com.email. - -[float] -[id="{beatname_lc}-gcp-labels"] -==== `labels` - -One or more labels to apply to the function. A label is a key-value pair that -helps you organize your Google Cloud resources. - -[float] -[id="{beatname_lc}-gcp-vpc_connector"] -==== `vpc_connector` - -A VPC connector that the function can connect to when sending requests to -resources in your VPC network. - -Use the format `projects/*/locations/*/connectors/*` or a fully qualified -URI. - -[float] -[id="{beatname_lc}-gcp-maximum_instances"] -==== `maximum_instances` - -The maximum instances that can be running at the same time. The default is -unlimited. - -[float] -[id="{beatname_lc}-gcp-triggers"] -===== `trigger` - -The trigger that will cause the function to execute. - -* If `type` is `pubsub`, specify the name of the Pub/Sub topic to watch for -messages. - -* If `type` is `storage`, specify the Cloud Storage bucket to watch for object -events. For `event_type`, specify the type of object event that will trigger the -function. See the https://cloud.google.com/functions/docs/calling/storage[Google Cloud -docs] for a list of available event types. - -[float] -[id="{beatname_lc}-gcp-keep_null"] -==== `keep_null` - -If `true`, fields with null values will be published in the output document. By -default, `keep_null` is `false`. - -[float] -[id="{beatname_lc}-gcp-fields"] -==== `fields` - -Optional fields that you can specify to add additional information to the -output. Fields can be scalar values, arrays, dictionaries, or any nested -combination of these. - -[float] -[id="{beatname_lc}-gcp-processors"] -==== `processors` - -Define custom processors for this function. For example, you can specify a -dissect processor to tokenize a string: - -[source,yaml] ----- -processors: - - dissect: - tokenizer: "%{key1} %{key2}" ----- diff --git a/x-pack/functionbeat/docs/configuring-howto.asciidoc b/x-pack/functionbeat/docs/configuring-howto.asciidoc index 3d72f9b5a55..fad0629261b 100644 --- a/x-pack/functionbeat/docs/configuring-howto.asciidoc +++ b/x-pack/functionbeat/docs/configuring-howto.asciidoc @@ -11,7 +11,6 @@ include::{libbeat-dir}/shared/configuring-intro.asciidoc[] * <> -* <> * <> * <> * <> @@ -28,8 +27,6 @@ include::{libbeat-dir}/shared/configuring-intro.asciidoc[] include::./config-options-aws.asciidoc[] -include::./config-options-gcp.asciidoc[] - include::./general-options.asciidoc[] [role="xpack"] diff --git a/x-pack/functionbeat/docs/getting-started.asciidoc b/x-pack/functionbeat/docs/getting-started.asciidoc index 2ba0caf85ed..ff5700cf71d 100644 --- a/x-pack/functionbeat/docs/getting-started.asciidoc +++ b/x-pack/functionbeat/docs/getting-started.asciidoc @@ -75,33 +75,6 @@ function. + See <> for more examples. -* *Google cloud example*: This example configures a function called -`storage` that collects log events from Google Cloud Storage. When the specified -event type occurs on the Cloud Storage bucket, the cloud function executes and -sends events to the configured output: -+ -["source","sh",subs="attributes"] ----- -functionbeat.provider.gcp.location_id: "europe-west2" -functionbeat.provider.gcp.project_id: "my-project-123456" -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" <1> -functionbeat.provider.gcp.functions: - - name: storage <2> - enabled: true - type: storage - description: "Google Cloud Function for Cloud Storage" - trigger: - resource: "projects/my-project/buckets/my-storage" - event_type: "google.storage.object.finalize" ----- -<1> The name of the GCP storage bucket where the function artifacts will be -deployed. -<2> Details about the function you want to deploy, including the name of the -function, the type of resource to monitor, and the resource event that triggers -the function. -+ -See <> for more examples. - -- include::{libbeat-dir}/shared/config-check.asciidoc[] @@ -176,41 +149,6 @@ If deployment fails, see <> for help troubleshooting. :fnexample!: -[float] -[[deploy-to-gcp]] -==== Deploy to Google Cloud Platform - -beta[] - -. In Google Cloud, create a service account that has these required roles: -+ --- -include::iam-permissions.asciidoc[tag=gcp-roles-deployment] --- -+ -See the https://cloud.google.com/docs/authentication/getting-started[Google -Cloud documentation] for more information about creating a service account. - -. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to point to -the JSON file that contains your service account key. For example: -+ --- -include::tab-widgets/credentials-google-widget.asciidoc[] --- - -. Deploy the cloud functions. -+ -For example, the following command deploys a function called `storage`: -+ --- -include::tab-widgets/deploy-google-widget.asciidoc[] --- -+ -The function is deployed to Google Cloud Platform and ready to send events -to the configured output. -+ -If deployment fails, see <> for help troubleshooting. - [float] [[view-data]] === Step 6: View your data in {kib} diff --git a/x-pack/functionbeat/docs/iam-permissions.asciidoc b/x-pack/functionbeat/docs/iam-permissions.asciidoc index 25dc16a236c..9b09d4d9a4e 100644 --- a/x-pack/functionbeat/docs/iam-permissions.asciidoc +++ b/x-pack/functionbeat/docs/iam-permissions.asciidoc @@ -10,7 +10,6 @@ This section describes the minimum privileges or roles required to deploy functions to your cloud provider: * <> -* <> [[iam-permissions-aws]] @@ -130,17 +129,3 @@ function that reads from SQS queues or Kinesis data streams. ] } ---- - -[[iam-permissions-gcp]] -===== Roles required by Google Cloud Platform - -The following roles are required to deploy Cloud Functions to Google Cloud -Platform: - -// tag::gcp-roles-deployment[] -* Cloud Functions Developer -* Cloud Functions Service Agent -* Service Account User -* Storage Admin -* Storage Object Admin -// end::gcp-roles-deployment[] diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 5531351df49..e5ccc87ba83 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -296,114 +296,6 @@ functionbeat.provider.aws.functions: # Set to true to publish fields with null values in events. #keep_null: false -# Configure functions to run on Google Cloud Platform, currently we assume that the credentials -# are present in the environment to correctly create the function when using the CLI. -# -# Configure which region your project is located in. -functionbeat.provider.gcp.location_id: "europe-west2" -# Configure which Google Cloud project to deploy your functions. -functionbeat.provider.gcp.project_id: "my-project-123456" -# Configure the Google Cloud Storage we should upload the function artifact. -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" - -functionbeat.provider.gcp.functions: - # Define the list of function availables, each function required to have a unique name. - # Create a function that accepts events coming from Google Pub/Sub. - - name: pubsub - enabled: false - type: pubsub - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Pub/Sub" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - trigger: - resource: "projects/_/pubsub/myPubSub" - #service: "pubsub.googleapis.com" - - # Set to true to publish fields with null values in events. - #keep_null: false - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" - - # Create a function that accepts events coming from Google Cloud Storage. - - name: storage - enabled: false - type: storage - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Cloud Storage" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - trigger: - resource: "projects/my-project/buckets/my-storage" - #event_type: "google.storage.object.finalize" - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - # Set to true to publish fields with null values in events. - #keep_null: false - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" - # ================================== General =================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index 3575f5e1d81..63b2b56bdcb 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -251,107 +251,6 @@ functionbeat.provider.aws.functions: # Default is 1. #parallelization_factor: 1 -# Configure functions to run on Google Cloud Platform, currently we assume that the credentials -# are present in the environment to correctly create the function when using the CLI. -# -# Configure which region your project is located in. -functionbeat.provider.gcp.location_id: "europe-west2" -# Configure which Google Cloud project to deploy your functions. -functionbeat.provider.gcp.project_id: "my-project-123456" -# Configure the Google Cloud Storage we should upload the function artifact. -functionbeat.provider.gcp.storage_name: "functionbeat-deploy" - -functionbeat.provider.gcp.functions: - # Define the list of function availables, each function required to have a unique name. - # Create a function that accepts events coming from Google Pub/Sub. - - name: pubsub - enabled: false - type: pubsub - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Pub/Sub" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - trigger: - resource: "projects/_/pubsub/myPubSub" - #service: "pubsub.googleapis.com" - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" - - # Create a function that accepts events coming from Google Cloud Storage. - - name: storage - enabled: false - type: storage - - # Description of the method to help identify them when you run multiples functions. - description: "Google Cloud Function for Cloud Storage" - - # The maximum memory allocated for this function. - # Default is 256MB. - #memory_size: 256MB - - # Execution timeout in seconds. If the function does not finish in time, - # it is considered failed and terminated. Default is 60s. - #timeout: 60s - - # Email of the service account of the function. Defaults to {projectid}@appspot.gserviceaccount.com - #service_account_email: {projectid}@appspot.gserviceaccount.com - - trigger: - resource: "projects/my-project/buckets/my-storage" - #event_type: "google.storage.object.finalize" - - # Labels of the function. - #labels: - # mylabel: label - - # VPC Connector this function can connect to. - # Format: projects/*/locations/*/connectors/* or fully-qualified URI - #vpc_connector: "" - - # Number of maximum instances running at the same time. Default is unlimited. - #maximum_instances: 0 - - # Optional fields that you can specify to add additional information to the - # output. Fields can be scalar values, arrays, dictionaries, or any nested - # combination of these. - #fields: - # env: staging - - # Define custom processors for this function. - #processors: - # - dissect: - # tokenizer: "%{key1} %{key2}" -#==================== Elasticsearch template setting ========================== - -setup.template.settings: - index.number_of_shards: 1 - #index.codec: best_compression - #_source.enabled: false # ================================== General =================================== diff --git a/x-pack/functionbeat/include/feature.go b/x-pack/functionbeat/include/feature.go index f66f8532f74..1770a05ef5a 100644 --- a/x-pack/functionbeat/include/feature.go +++ b/x-pack/functionbeat/include/feature.go @@ -7,14 +7,12 @@ package include import ( "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/x-pack/functionbeat/manager/aws" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/gcp" "github.com/elastic/beats/v7/x-pack/functionbeat/provider/local/local" ) // Bundle feature enabled. var Bundle = feature.MustBundle( aws.Bundle, - gcp.Bundle, local.Bundle, ) diff --git a/x-pack/functionbeat/magefile.go b/x-pack/functionbeat/magefile.go index afe34d59dcf..7a755bc4b87 100644 --- a/x-pack/functionbeat/magefile.go +++ b/x-pack/functionbeat/magefile.go @@ -167,11 +167,7 @@ func BuildPkgForFunctions() error { err := os.RemoveAll("pkg") filesToCopy := map[string]string{ - filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), - filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), - filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), - filepath.Join("provider", "gcp", "build", "pubsub", "vendor"): filepath.Join("pkg", "pubsub", "vendor"), - filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"), + filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), } for src, dest := range filesToCopy { c := &devtools.CopyTask{ @@ -186,29 +182,6 @@ func BuildPkgForFunctions() error { return nil } -// TestGCPFunctions are used by the CI to test if the GCP functions can be built with -// the selected Go version. -// The version is 1.13.1 (Ref: https://cloud.google.com/functions/docs/concepts/go-runtime) -func TestGCPFunctions() error { - for _, f := range []string{"pubsub", "storage"} { - params := devtools.DefaultBuildArgs() - inputFiles := filepath.Join("provider", "gcp", f, f+".go") - params.InputFiles = []string{inputFiles} - params.Name = f - params.CGO = false - params.Env = map[string]string{ - "GOOS": "linux", - "GOARCH": "amd64", - } - - err := devtools.Build(params) - if err != nil { - return fmt.Errorf("error while building %s for GCP: %+v", f, err) - } - } - return nil -} - // BuildSystemTestBinary build a binary for testing that is instrumented for // testing and measuring code coverage. The binary is only instrumented for // coverage when TEST_COVERAGE=true (default is false). diff --git a/x-pack/functionbeat/manager/gcp/cli_manager.go b/x-pack/functionbeat/manager/gcp/cli_manager.go deleted file mode 100644 index d286f1e465a..00000000000 --- a/x-pack/functionbeat/manager/gcp/cli_manager.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "strings" - - "github.com/pkg/errors" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/core" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" - fngcp "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" -) - -type installer interface { - Config() *fngcp.FunctionConfig - Name() string -} - -// CLIManager interacts with Google Cloud to deploy, update or remove a function. -type CLIManager struct { - templateBuilder *defaultTemplateBuilder - location string - tokenSrc oauth2.TokenSource - log *logp.Logger - config *Config -} - -// Deploy uploads the function to GCP. -func (c *CLIManager) Deploy(name string) error { - c.log.Debugf("Deploying function: %s", name) - defer c.log.Debugf("Deploy finish for function '%s'", name) - - err := c.deploy(false, name) - if err != nil { - return err - } - - c.log.Debugf("Successfully created function '%s'", name) - return nil -} - -// Update updates the function. -func (c *CLIManager) Update(name string) error { - c.log.Debugf("Starting updating function '%s'", name) - defer c.log.Debugf("Update complete for function '%s'", name) - - err := c.deploy(true, name) - if err != nil { - return err - } - - c.log.Debugf("Successfully updated function: '%s'", name) - return nil -} - -// deploy uploads to bucket and creates a function on GCP. -func (c *CLIManager) deploy(update bool, name string) error { - functionData, err := c.templateBuilder.execute(name) - if err != nil { - return err - } - - executer := executor.NewExecutor(c.log) - executer.Add(newOpEnsureBucket(c.log, c.config)) - executer.Add(newOpUploadToBucket(c.log, c.config, name, functionData.raw)) - - token, err := c.getTokenSrc() - if err != nil { - return err - } - - ctx := &functionContext{} - if update { - executer.Add(newOpUpdateFunction(ctx, c.log, token, functionData.function.Name, functionData.function)) - } else { - executer.Add(newOpCreateFunction(ctx, c.log, token, c.location, name, functionData.function)) - } - - executer.Add(newOpWaitForFunction(ctx, c.log, token)) - - if err := executer.Execute(nil); err != nil { - if rollbackErr := executer.Rollback(nil); rollbackErr != nil { - return errors.Wrapf(err, "could not rollback, error: %s", rollbackErr) - } - return err - } - return nil -} - -// Remove removes a stack and unregister any resources created. -func (c *CLIManager) Remove(name string) error { - c.log.Debugf("Removing function: %s", name) - defer c.log.Debugf("Removal of function '%s' complete", name) - - functionData, err := c.templateBuilder.execute(name) - if err != nil { - return err - } - - token, err := c.getTokenSrc() - if err != nil { - return err - } - - ctx := &functionContext{} - executer := executor.NewExecutor(c.log) - executer.Add(newOpDeleteFunction(ctx, c.log, c.location, functionData.function.Name, token)) - executer.Add(newOpDeleteFromBucket(c.log, c.config, name)) - - if err := executer.Execute(nil); err != nil { - if rollbackErr := executer.Rollback(nil); rollbackErr != nil { - return errors.Wrapf(err, "could not rollback, error: %s", rollbackErr) - } - return err - } - - c.log.Debugf("Successfully deleted function: '%s'", name) - return nil -} - -// Export prints the exported function data. -func (c *CLIManager) Export(name string) error { - tmpl, err := c.templateBuilder.RawTemplate(name) - if err != nil { - return err - } - fmt.Println(tmpl) - - return nil -} - -// Package packages functions for GCP. -func (c *CLIManager) Package(outputPattern string) error { - resources := zipResources() - for providerSuffix, r := range resources { - content, err := core.MakeZip(packageUncompressedLimit, packageCompressedLimit, r) - if err != nil { - return err - } - - output := strings.ReplaceAll(outputPattern, "{{.Provider}}", providerSuffix) - err = ioutil.WriteFile(output, content, 0644) - if err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "Generated package for provider %s at: %s\n", providerSuffix, output) - } - return nil -} - -func (c *CLIManager) getTokenSrc() (oauth2.TokenSource, error) { - if c.tokenSrc != nil { - return c.tokenSrc, nil - } - - var err error - c.tokenSrc, err = google.DefaultTokenSource(context.Background(), "https://www.googleapis.com/auth/cloud-platform") - if err != nil { - return nil, fmt.Errorf("error while creating CLIManager: %+v", err) - } - - return c.tokenSrc, nil -} - -// NewCLI returns the interface to manage functions on Google Cloud Platform. -func NewCLI( - log *logp.Logger, - cfg *common.Config, - provider provider.Provider, -) (provider.CLIManager, error) { - config := &Config{} - if err := cfg.Unpack(config); err != nil { - return nil, err - } - - builder, err := provider.TemplateBuilder() - if err != nil { - return nil, err - } - templateBuilder, ok := builder.(*defaultTemplateBuilder) - if !ok { - return nil, fmt.Errorf("not defaultTemplateBuilder") - } - - location := fmt.Sprintf(locationTemplate, config.ProjectID, config.Location) - - return &CLIManager{ - config: config, - log: logp.NewLogger("gcp"), - location: location, - templateBuilder: templateBuilder, - }, nil -} diff --git a/x-pack/functionbeat/manager/gcp/cli_manager_test.go b/x-pack/functionbeat/manager/gcp/cli_manager_test.go deleted file mode 100644 index 63d3e1f445b..00000000000 --- a/x-pack/functionbeat/manager/gcp/cli_manager_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp diff --git a/x-pack/functionbeat/manager/gcp/config.go b/x-pack/functionbeat/manager/gcp/config.go deleted file mode 100644 index d6fcd9b24e0..00000000000 --- a/x-pack/functionbeat/manager/gcp/config.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -// Config exposes the configuration options of the GCP provider. -type Config struct { - Location string `config:"location_id" validate:"required"` - ProjectID string `config:"project_id" validate:"required"` - FunctionStorage string `config:"storage_name" validate:"required"` -} diff --git a/x-pack/functionbeat/manager/gcp/function_context.go b/x-pack/functionbeat/manager/gcp/function_context.go deleted file mode 100644 index 9a558214ee8..00000000000 --- a/x-pack/functionbeat/manager/gcp/function_context.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "errors" -) - -var ( - errWrongContext = errors.New("invalid type, expecting function context") - errMissingFunctionName = errors.New("missing function operation name") -) - -type functionContext struct { - name string -} diff --git a/x-pack/functionbeat/manager/gcp/gcp.go b/x-pack/functionbeat/manager/gcp/gcp.go deleted file mode 100644 index c33ae2493a6..00000000000 --- a/x-pack/functionbeat/manager/gcp/gcp.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" -) - -// Bundle exposes the trigger supported by the gcp provider. -var Bundle = provider.MustCreate( - "gcp", - provider.NewDefaultProvider("gcp", NewCLI, NewTemplateBuilder), - feature.MakeDetails("Google Cloud Functions", "listen to events on Google Cloud", feature.Stable), -).MustAddFunction("pubsub", - gcp.NewPubSub, - gcp.PubSubDetails(), -).MustAddFunction("storage", - gcp.NewStorage, - gcp.StorageDetails(), -).Bundle() diff --git a/x-pack/functionbeat/manager/gcp/op_create_function.go b/x-pack/functionbeat/manager/gcp/op_create_function.go deleted file mode 100644 index cf970a77091..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_create_function.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "golang.org/x/oauth2" - cloudfunctions "google.golang.org/api/cloudfunctions/v1" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opCreateFunction struct { - ctx *functionContext - log *logp.Logger - tokenSrc oauth2.TokenSource - location string - name string - function *cloudfunctions.CloudFunction -} - -func newOpCreateFunction( - ctx *functionContext, - log *logp.Logger, - tokenSrc oauth2.TokenSource, - location string, - name string, - f *cloudfunctions.CloudFunction, -) *opCreateFunction { - return &opCreateFunction{ - ctx: ctx, - log: log, - tokenSrc: tokenSrc, - name: name, - location: location, - function: f, - } -} - -// Execute creates a function from the zip uploaded. -func (o *opCreateFunction) Execute(_ executor.Context) error { - o.log.Debugf("Creating function %s at %s", o.function.Name, o.function.SourceArchiveUrl) - - client := oauth2.NewClient(context.TODO(), o.tokenSrc) - svc, err := cloudfunctions.New(client) - if err != nil { - return fmt.Errorf("error while creating cloud functions service: %+v", err) - } - - functionSvc := cloudfunctions.NewProjectsLocationsFunctionsService(svc) - operation, err := functionSvc.Create(o.location, o.function).Context(context.TODO()).Do() - if err != nil { - return fmt.Errorf("error while creating function: %+v", err) - } - - o.ctx.name = operation.Name - - if operation.Done { - o.log.Debugf("Function %s created successfully", o.function.Name) - } else { - o.log.Debugf("Operation '%s' is in progress to create function %s", operation.Name, o.function.Name) - } - - return nil -} - -// Rollback removes the deployed function. -func (o *opCreateFunction) Rollback(_ executor.Context) error { - return newOpDeleteFunction(o.ctx, o.log, o.location, o.function.Name, o.tokenSrc).Execute(nil) -} diff --git a/x-pack/functionbeat/manager/gcp/op_delete_file_bucket.go b/x-pack/functionbeat/manager/gcp/op_delete_file_bucket.go deleted file mode 100644 index 50841bd7f63..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_delete_file_bucket.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "cloud.google.com/go/storage" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opDeleteFromBucket struct { - log *logp.Logger - config *Config - name string -} - -func newOpDeleteFromBucket(log *logp.Logger, config *Config, name string) *opDeleteFromBucket { - return &opDeleteFromBucket{ - log: log, - config: config, - name: name, - } -} - -// Execute removes the function from the bucket. -// storage.objects.delete permission is required. -func (o *opDeleteFromBucket) Execute(_ executor.Context) error { - o.log.Debugf("Removing file '%s' from bucket '%s'", o.name, o.config.FunctionStorage) - - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("could not create storage client: %+v", err) - } - - err = client.Bucket(o.config.FunctionStorage).Object(o.name).Delete(ctx) - if err != nil { - return err - } - - o.log.Debugf("Successfully removed function '%s' from bucket '%s'", o.name, o.config.FunctionStorage) - return nil -} diff --git a/x-pack/functionbeat/manager/gcp/op_delete_function.go b/x-pack/functionbeat/manager/gcp/op_delete_function.go deleted file mode 100644 index 97b8b25cf11..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_delete_function.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "golang.org/x/oauth2" - cloudfunctions "google.golang.org/api/cloudfunctions/v1" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opDeleteFunction struct { - ctx *functionContext - log *logp.Logger - location string - name string - tokenSrc oauth2.TokenSource -} - -func newOpDeleteFunction( - ctx *functionContext, - log *logp.Logger, - location string, - name string, - tokenSrc oauth2.TokenSource, -) *opDeleteFunction { - return &opDeleteFunction{ - ctx: ctx, - log: log, - location: location, - name: name, - tokenSrc: tokenSrc, - } -} - -// Execute creates a function from the zip uploaded. -func (o *opDeleteFunction) Execute(_ executor.Context) error { - client := oauth2.NewClient(context.TODO(), o.tokenSrc) - svc, err := cloudfunctions.New(client) - if err != nil { - return fmt.Errorf("error while creating cloud functions service: %+v", err) - } - - functionSvc := cloudfunctions.NewProjectsLocationsFunctionsService(svc) - operation, err := functionSvc.Delete(o.name).Context(context.TODO()).Do() - if err != nil { - return fmt.Errorf("error while removing function %s: %+v", o.name, err) - } - - o.ctx.name = operation.Name - - if operation.Done { - o.log.Debugf("Function %s removed successfully", o.name) - } - - return nil -} - -// Rollback -func (o *opDeleteFunction) Rollback(_ executor.Context) error { - return nil -} diff --git a/x-pack/functionbeat/manager/gcp/op_ensure_bucket.go b/x-pack/functionbeat/manager/gcp/op_ensure_bucket.go deleted file mode 100644 index 06da4fb446f..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_ensure_bucket.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "cloud.google.com/go/storage" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opEnsureBucket struct { - log *logp.Logger - config *Config - created bool -} - -func newOpEnsureBucket(log *logp.Logger, cfg *Config) *opEnsureBucket { - return &opEnsureBucket{log: log, config: cfg, created: false} -} - -func (o *opEnsureBucket) Execute(_ executor.Context) error { - o.log.Debugf("Verifying presence of Cloud Storage bucket: '%s'", o.config.FunctionStorage) - - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return err - } - - bucket := client.Bucket(o.config.FunctionStorage) - attrs, err := bucket.Attrs(ctx) - if err == storage.ErrBucketNotExist { - berr := bucket.Create(ctx, o.config.ProjectID, nil) - if berr != nil { - return fmt.Errorf("cannot create bucket for function: %+v", berr) - } - o.created = true - o.log.Debugf("Cloud Storage bucket created with name '%s', attributes: %+v", o.config.FunctionStorage, attrs) - return nil - } - - if err != nil { - return fmt.Errorf("cannot get info on bucket: %+v", err) - } - - o.log.Debugf("Cloud Storage bucket exists with name '%s', attributes: %+v", o.config.FunctionStorage, attrs) - return nil -} - -func (o *opEnsureBucket) Rollback(_ executor.Context) error { - if o.created { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return err - } - err = client.Bucket(o.config.FunctionStorage).Delete(ctx) - if err != nil { - return err - } - } - return nil -} diff --git a/x-pack/functionbeat/manager/gcp/op_update_function.go b/x-pack/functionbeat/manager/gcp/op_update_function.go deleted file mode 100644 index ca70af1aad8..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_update_function.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "golang.org/x/oauth2" - cloudfunctions "google.golang.org/api/cloudfunctions/v1" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opUpdateFunction struct { - ctx *functionContext - log *logp.Logger - tokenSrc oauth2.TokenSource - name string - function *cloudfunctions.CloudFunction -} - -func newOpUpdateFunction( - ctx *functionContext, - log *logp.Logger, - tokenSrc oauth2.TokenSource, - name string, - f *cloudfunctions.CloudFunction, -) *opUpdateFunction { - return &opUpdateFunction{ - ctx: ctx, - log: log, - tokenSrc: tokenSrc, - name: name, - function: f, - } -} - -// Execute updates an existing function. -func (o *opUpdateFunction) Execute(_ executor.Context) error { - o.log.Debugf("Updating function %s at %s", o.function.Name, o.function.SourceArchiveUrl) - - client := oauth2.NewClient(context.TODO(), o.tokenSrc) - svc, err := cloudfunctions.New(client) - if err != nil { - return fmt.Errorf("error while creating cloud functions service: %+v", err) - } - - functionSvc := cloudfunctions.NewProjectsLocationsFunctionsService(svc) - operation, err := functionSvc.Patch(o.name, o.function).Context(context.TODO()).Do() - if err != nil { - return fmt.Errorf("error while updating function: %+v", err) - } - - o.ctx.name = operation.Name - - if operation.Done { - o.log.Debugf("Function %s updated successfully", o.function.Name) - } else { - o.log.Debugf("Operation '%s' is in progress to update function %s", operation.Name, o.function.Name) - } - - return nil -} - -// Rollback updates the deployed function. -func (o *opUpdateFunction) Rollback(_ executor.Context) error { - return nil -} diff --git a/x-pack/functionbeat/manager/gcp/op_upload_to_bucket.go b/x-pack/functionbeat/manager/gcp/op_upload_to_bucket.go deleted file mode 100644 index a05a16c7b71..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_upload_to_bucket.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "cloud.google.com/go/storage" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -type opUploadToBucket struct { - log *logp.Logger - config *Config - name string - raw []byte -} - -func newOpUploadToBucket(log *logp.Logger, config *Config, name string, raw []byte) *opUploadToBucket { - return &opUploadToBucket{ - log: log, - config: config, - name: name, - raw: raw, - } -} - -// Execute loads function to bucket. -// If function needs to be overwritten, storage.objects.delete permission is required. -func (o *opUploadToBucket) Execute(_ executor.Context) error { - o.log.Debugf("Uploading file '%s' to bucket '%s' with size %d bytes", o.name, o.config.FunctionStorage, len(o.raw)) - - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return fmt.Errorf("could not create storage client: %+v", err) - } - - w := client.Bucket(o.config.FunctionStorage).Object(o.name).NewWriter(ctx) - w.ContentType = "text/plain" - _, err = w.Write(o.raw) - if err != nil { - return fmt.Errorf("error while writing function: %+v", err) - } - err = w.Close() - if err != nil { - return fmt.Errorf("error while closing writer: %+v", err) - } - - o.log.Debugf("Upload to bucket was successful") - - return nil -} - -// Rollback removes the loaded archive. -func (o *opUploadToBucket) Rollback(ctx executor.Context) error { - err := newOpDeleteFromBucket(o.log, o.config, o.name).Execute(ctx) - if err != nil { - o.log.Debugf("Fail to delete file from bucket, error: %+v", err) - } - return nil -} diff --git a/x-pack/functionbeat/manager/gcp/op_wait_for_function.go b/x-pack/functionbeat/manager/gcp/op_wait_for_function.go deleted file mode 100644 index 94cf2dcbc0d..00000000000 --- a/x-pack/functionbeat/manager/gcp/op_wait_for_function.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - "time" - - "golang.org/x/oauth2" - cloudfunctions "google.golang.org/api/cloudfunctions/v1" - - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/executor" -) - -var periodicCheck = 5 * time.Second - -type opWaitForFunction struct { - ctx *functionContext - log *logp.Logger - tokenSrc oauth2.TokenSource -} - -func newOpWaitForFunction(ctx *functionContext, log *logp.Logger, tokenSrc oauth2.TokenSource) *opWaitForFunction { - return &opWaitForFunction{ - ctx: ctx, - log: log, - tokenSrc: tokenSrc, - } -} - -func (o *opWaitForFunction) Execute(_ executor.Context) error { - if o.ctx.name == "" { - return errMissingFunctionName - } - - client := oauth2.NewClient(context.TODO(), o.tokenSrc) - svc, err := cloudfunctions.New(client) - if err != nil { - return fmt.Errorf("error while creating cloud functions service: %+v", err) - } - - opSvc := cloudfunctions.NewOperationsService(svc) - for { - op, err := opSvc.Get(o.ctx.name).Context(context.Background()).Do() - if err != nil { - return err - } - - if op.Done { - if op.Error != nil { - return fmt.Errorf("error while creating function (code: %d):\n%s", op.Error.Code, op.Error.Message) - } - o.log.Debugf("Successfully deployed function") - return nil - } - - o.log.Debugf("Operation %s has not finished yet. Retrying in 5 seconds.", o.ctx.name) - - timer := time.NewTimer(periodicCheck) - <-timer.C - } -} diff --git a/x-pack/functionbeat/manager/gcp/template_builder.go b/x-pack/functionbeat/manager/gcp/template_builder.go deleted file mode 100644 index c6f1292ce15..00000000000 --- a/x-pack/functionbeat/manager/gcp/template_builder.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "errors" - "fmt" - "path/filepath" - - cloudfunctions "google.golang.org/api/cloudfunctions/v1" - yaml "gopkg.in/yaml.v2" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/core" - "github.com/elastic/beats/v7/x-pack/functionbeat/manager/core/bundle" - fngcp "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" -) - -const ( - runtime = "go113" // Golang 1.13 - archiveURL = "gs://%s/%s" // path to the function archive - locationTemplate = "projects/%s/locations/%s" // full name of the location - functionName = locationTemplate + "/functions/%s" // full name of the functions - - // Package size limits for GCP provider - // Ref: https://cloud.google.com/functions/quotas - packageCompressedLimit = 100 * 1000 * 1000 // 100MB - packageUncompressedLimit = 500 * 1000 * 1000 // 500MB -) - -// defaultTemplateBuilder builds request object when deploying Functionbeat using -// the command deploy. -type defaultTemplateBuilder struct { - provider provider.Provider - log *logp.Logger - gcpConfig *Config -} - -type functionData struct { - raw []byte - function *cloudfunctions.CloudFunction -} - -// NewTemplateBuilder returns the requested template builder -func NewTemplateBuilder(log *logp.Logger, cfg *common.Config, p provider.Provider) (provider.TemplateBuilder, error) { - gcpCfg := &Config{} - err := cfg.Unpack(gcpCfg) - if err != nil { - return &defaultTemplateBuilder{}, err - } - - return &defaultTemplateBuilder{log: log, gcpConfig: gcpCfg, provider: p}, nil -} - -func (d *defaultTemplateBuilder) execute(name string) (*functionData, error) { - d.log.Debug("Compressing all assets into an artifact") - - fn, err := findFunction(d.provider, name) - if err != nil { - return nil, err - } - - resources := zipResourcesOfFunc(fn.Name()) - raw, err := core.MakeZip(packageUncompressedLimit, packageCompressedLimit, resources) - if err != nil { - return nil, err - } - - d.log.Debugf("Compression is successful (zip size: %d bytes)", len(raw)) - - return &functionData{ - raw: raw, - function: d.cloudFunction(name, fn.Config()), - }, nil -} - -func findFunction(p provider.Provider, name string) (installer, error) { - fn, err := p.FindFunctionByName(name) - if err != nil { - return nil, err - } - - function, ok := fn.(installer) - if !ok { - return nil, errors.New("incompatible type received, expecting: 'functionManager'") - } - - return function, nil -} - -func (d *defaultTemplateBuilder) cloudFunction(name string, config *fngcp.FunctionConfig) *cloudfunctions.CloudFunction { - fnName := fmt.Sprintf(functionName, d.gcpConfig.ProjectID, d.gcpConfig.Location, name) - sourceArchiveURL := fmt.Sprintf(archiveURL, d.gcpConfig.FunctionStorage, name) - - return &cloudfunctions.CloudFunction{ - Name: fnName, - Description: config.Description, - EntryPoint: config.EntryPoint(), - EnvironmentVariables: map[string]string{ - "ENABLED_FUNCTIONS": name, - "BEAT_STRICT_PERMS": "false", - }, - EventTrigger: &cloudfunctions.EventTrigger{ - EventType: config.Trigger.EventType, - Resource: config.Trigger.Resource, - Service: config.Trigger.Service, - }, - Labels: config.Labels, - MaxInstances: int64(config.MaxInstances), - Runtime: runtime, - ServiceAccountEmail: config.ServiceAccountEmail, - SourceArchiveUrl: sourceArchiveURL, - Timeout: config.Timeout, - VpcConnector: config.VPCConnector, - } -} - -// RawTemplate returns the JSON to POST to the endpoint. -func (d *defaultTemplateBuilder) RawTemplate(name string) (string, error) { - fn, err := findFunction(d.provider, name) - if err != nil { - return "", err - } - config := fn.Config() - - properties := common.MapStr{ - "codeLocation": "pkg/" + fn.Name(), - "codeBucket": d.gcpConfig.FunctionStorage, - "codeBucketObject": "functionbeat.zip", - "location": d.gcpConfig.Location, - "runtime": runtime, - "entryPoint": config.EntryPoint(), - "eventTrigger": config.Trigger, - "environmentVariables": common.MapStr{ - "ENABLED_FUNCTIONS": name, - "BEAT_STRICT_PERMS": false, - }, - } - - if config.Timeout != "" { - properties["timeout"] = config.Timeout - } - if config.MemorySize != "" { - properties["availableMemoryMb"] = config.MemorySize - } - if len(config.ServiceAccountEmail) > 0 { - properties["serviceAccountEmail"] = config.ServiceAccountEmail - } - if len(config.Labels) > 0 { - properties["labels"] = config.Labels - } - if config.MaxInstances > 0 { - properties["maxInstances"] = config.MaxInstances - } - if len(config.VPCConnector) > 0 { - properties["vpcConnector"] = config.VPCConnector - } - - output := common.MapStr{ - "resources": []common.MapStr{ - common.MapStr{ - "name": fmt.Sprintf(functionName, d.gcpConfig.ProjectID, d.gcpConfig.Location, name), - "type": "google.cloud.functions.v1.CloudFunction", - "properties": properties, - }, - }, - } - - yamlBytes, err := yaml.Marshal(output) - return string(yamlBytes), err -} - -func zipResources() map[string][]bundle.Resource { - functions, err := provider.ListFunctions("gcp") - if err != nil { - fmt.Println(err) - return nil - } - - resources := make(map[string][]bundle.Resource) - for _, f := range functions { - resources["gcp-"+f] = zipResourcesOfFunc(f) - } - return resources -} - -func zipResourcesOfFunc(typeName string) []bundle.Resource { - root := filepath.Join("pkg", typeName) - vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644) - return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}) -} diff --git a/x-pack/functionbeat/provider/gcp/gcp/config.go b/x-pack/functionbeat/provider/gcp/gcp/config.go deleted file mode 100644 index 0af370edcf2..00000000000 --- a/x-pack/functionbeat/provider/gcp/gcp/config.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "fmt" -) - -// FunctionConfig stores the configuration of a Google Cloud Function -type FunctionConfig struct { - Description string `config:"description"` - MemorySize string `config:"memory_size"` - Timeout string `config:"timeout"` - ServiceAccountEmail string `config:"service_account_email"` - Labels map[string]string `config:"labels"` - VPCConnector string `config:"vpc_connector"` - MaxInstances int `config:"maximum_instances"` - Trigger Trigger `config:"trigger" validate:"required"` - - entryPoint string -} - -// Trigger stores the configuration of the function trigger. -type Trigger struct { - EventType string `config:"event_type" json:"eventType"` - Resource string `config:"resource" validate:"required" json:"resource"` - Service string `config:"service" json:"service,omitempty" yaml:"service,omitempty"` -} - -func defaultPubSubFunctionConfig() *FunctionConfig { - return &FunctionConfig{ - Trigger: Trigger{ - EventType: "google.pubsub.topic.publish", - }, - entryPoint: "RunPubSub", - } -} - -func defaultStorageFunctionConfig() *FunctionConfig { - return &FunctionConfig{ - Trigger: Trigger{ - EventType: "google.storage.object.finalize", - }, - entryPoint: "RunCloudStorage", - } -} - -// Validate checks a function configuration. -func (c *FunctionConfig) Validate() error { - if c.entryPoint == "" { - return fmt.Errorf("entryPoint cannot be empty") - } - return nil -} - -// EntryPoint returns the name of the function to run on GCP. -func (c *FunctionConfig) EntryPoint() string { - return c.entryPoint -} diff --git a/x-pack/functionbeat/provider/gcp/gcp/pubsub.go b/x-pack/functionbeat/provider/gcp/gcp/pubsub.go deleted file mode 100644 index 19b45428bbd..00000000000 --- a/x-pack/functionbeat/provider/gcp/gcp/pubsub.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "cloud.google.com/go/functions/metadata" - "cloud.google.com/go/pubsub" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/core" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/telemetry" -) - -const ( - pubSubEventCtxStr = "pub_sub_event" -) - -// PubSub represents a Google Cloud function which reads event from Google Pub/Sub triggers. -type PubSub struct { - log *logp.Logger - config *FunctionConfig -} - -// PubSubEventKey is an alias to string -type PubSubEventKey string - -// NewPubSub returns a new function to read from Google Pub/Sub. -func NewPubSub(provider provider.Provider, cfg *common.Config) (provider.Function, error) { - config := defaultPubSubFunctionConfig() - err := cfg.Unpack(config) - if err != nil { - return &PubSub{}, err - } - - return &PubSub{ - log: logp.NewLogger("pubsub"), - config: config, - }, nil -} - -// PubSubEvent stores the context and the message from Google Pub/Sub. -type PubSubEvent struct { - Metadata *metadata.Metadata - Message pubsub.Message -} - -// NewPubSubContext creates a context from context and message returned from Google Pub/Sub. -func NewPubSubContext(beatCtx, ctx context.Context, m pubsub.Message) (context.Context, error) { - data, err := metadata.FromContext(ctx) - if err != nil { - return nil, err - } - e := PubSubEvent{ - Metadata: data, - Message: m, - } - - return context.WithValue(beatCtx, PubSubEventKey(pubSubEventCtxStr), e), nil -} - -// Run start -func (p *PubSub) Run(ctx context.Context, client core.Client, t telemetry.T) error { - t.AddTriggeredFunction() - - pubsubEvent, err := p.getEventDataFromContext(ctx) - if err != nil { - return err - } - event, err := transformPubSub(pubsubEvent.Metadata, pubsubEvent.Message) - if err := client.Publish(event); err != nil { - p.log.Errorf("error while publishing Pub/Sub event %+v", err) - return err - } - client.Wait() - - return nil -} - -func (p *PubSub) getEventDataFromContext(ctx context.Context) (PubSubEvent, error) { - iPubSubEvent := ctx.Value(PubSubEventKey(pubSubEventCtxStr)) - if iPubSubEvent == nil { - return PubSubEvent{}, fmt.Errorf("no pub_sub_event in context") - } - event, ok := iPubSubEvent.(PubSubEvent) - if !ok { - return PubSubEvent{}, fmt.Errorf("not PubSubEvent: %+v", iPubSubEvent) - } - - return event, nil -} - -// PubSubDetails returns the details of the feature. -func PubSubDetails() feature.Details { - return feature.MakeDetails("Google Pub/Sub trigger", "receive messages from Google Pub/Sub.", feature.Stable) -} - -// Name returns the name of the function. -func (p *PubSub) Name() string { - return "pubsub" -} - -// Config returns the configuration to use when creating the function. -func (p *PubSub) Config() *FunctionConfig { - return p.config -} diff --git a/x-pack/functionbeat/provider/gcp/gcp/storage.go b/x-pack/functionbeat/provider/gcp/gcp/storage.go deleted file mode 100644 index 5b1d6e7f4e6..00000000000 --- a/x-pack/functionbeat/provider/gcp/gcp/storage.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "context" - "fmt" - - "cloud.google.com/go/functions/metadata" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/core" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/telemetry" -) - -const ( - storageEvtCtxStr = "storage_event" -) - -// Storage represents a Google Cloud function which reads event from Google Cloud Storage. -type Storage struct { - log *logp.Logger - config *FunctionConfig -} - -// StorageEventKey is an alias to string -type StorageEventKey string - -// StorageEventWithMeta stores the storage event received from Google Cloud Storage. -type StorageEventWithMeta struct { - Metadata *metadata.Metadata - Event StorageEvent -} - -// NewStorage returns a new function to read from Google Cloud Storage. -func NewStorage(provider provider.Provider, cfg *common.Config) (provider.Function, error) { - config := defaultStorageFunctionConfig() - err := cfg.Unpack(config) - if err != nil { - return &Storage{}, err - } - - return &Storage{ - log: logp.NewLogger("storage"), - config: config, - }, nil -} - -// NewStorageContext creates a context from context and message returned from Google Cloud Storage. -func NewStorageContext(beatCtx, ctx context.Context, e StorageEvent) (context.Context, error) { - data, err := metadata.FromContext(ctx) - if err != nil { - return nil, err - } - - evt := StorageEventWithMeta{ - Metadata: data, - Event: e, - } - - return context.WithValue(beatCtx, StorageEventKey(storageEvtCtxStr), evt), nil -} - -// Run start -func (s *Storage) Run(ctx context.Context, client core.Client, t telemetry.T) error { - t.AddTriggeredFunction() - - evt, err := s.getEventDataFromContext(ctx) - if err != nil { - return err - } - event, err := transformStorage(evt.Metadata, evt.Event) - if err := client.Publish(event); err != nil { - s.log.Errorf("error while publishing Google Cloud Storage event %+v", err) - return err - } - client.Wait() - - return nil -} - -func (s *Storage) getEventDataFromContext(ctx context.Context) (StorageEventWithMeta, error) { - iEvt := ctx.Value(StorageEventKey(storageEvtCtxStr)) - if iEvt == nil { - return StorageEventWithMeta{}, fmt.Errorf("no storage_event in context") - } - evt, ok := iEvt.(StorageEventWithMeta) - if !ok { - return StorageEventWithMeta{}, fmt.Errorf("not StorageEvent: %+v", iEvt) - } - - return evt, nil -} - -// StorageDetails returns the details of the feature. -func StorageDetails() feature.Details { - return feature.MakeDetails("Google Cloud Storage trigger", "receive events from Google Cloud Storage.", feature.Stable) -} - -// Name returns the name of the function. -func (s *Storage) Name() string { - return "storage" -} - -// Config returns the configuration to use when creating the function. -func (s *Storage) Config() *FunctionConfig { - return s.config -} diff --git a/x-pack/functionbeat/provider/gcp/gcp/transformer.go b/x-pack/functionbeat/provider/gcp/gcp/transformer.go deleted file mode 100644 index beb4fa7f354..00000000000 --- a/x-pack/functionbeat/provider/gcp/gcp/transformer.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package gcp - -import ( - "time" - - "cloud.google.com/go/functions/metadata" - "cloud.google.com/go/pubsub" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common" -) - -// StorageEvent is the event from Google Cloud Storage -type StorageEvent struct { - Bucket string `json:"bucket"` - Name string `json:"name"` - Metageneration string `json:"metageneration"` - ResourceState string `json:"resourceState"` - Created time.Time `json:"timeCreated"` - Updated time.Time `json:"updated"` -} - -// transformPubSub takes a Pub/Sub message and context and transforms it into an event. -func transformPubSub(mData *metadata.Metadata, msg pubsub.Message) (beat.Event, error) { - return beat.Event{ - Timestamp: mData.Timestamp, - Fields: common.MapStr{ - "event": common.MapStr{ - "kind": "event", - }, - "cloud": common.MapStr{ - "provider": "gcp", - }, - "read_timestamp": time.Now(), - "message": string(msg.Data), - "attributes": msg.Attributes, - "id": mData.EventID, - "resource": common.MapStr{ - "service": mData.Resource.Service, - "name": mData.Resource.Name, - "event_type": mData.Resource.Type, - }, - }, - }, nil -} - -// transformStorage takes a Cloud Storage object and transforms it into an event. -func transformStorage(mData *metadata.Metadata, evt StorageEvent) (beat.Event, error) { - - return beat.Event{ - Timestamp: mData.Timestamp, - Fields: common.MapStr{ - "event": common.MapStr{ - "kind": "event", - "category": []string{"file"}, - "type": []string{"info"}, - }, - "cloud": common.MapStr{ - "provider": "gcp", - }, - "read_timestamp": time.Now(), - "id": mData.EventID, - "resource": common.MapStr{ - "service": mData.Resource.Service, - "name": mData.Resource.Name, - "event_type": mData.Resource.Type, - "state": evt.ResourceState, - }, - "storage_bucket": evt.Bucket, - "file": common.MapStr{ - "name": evt.Name, - "mtime": evt.Updated, - "ctime": evt.Updated, - "created": evt.Created, - }, - "meta-generation": evt.Metageneration, - }, - }, nil -} diff --git a/x-pack/functionbeat/provider/gcp/include/feature.go b/x-pack/functionbeat/provider/gcp/include/feature.go deleted file mode 100644 index 7af761e82de..00000000000 --- a/x-pack/functionbeat/provider/gcp/include/feature.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package include - -import ( - "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" - "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" -) - -// Bundle exposes the trigger supported by the GCP provider. -var bundle = provider.MustCreate( - "gcp", - provider.NewDefaultProvider("gcp", provider.NewNullCli, provider.NewNullTemplateBuilder), - feature.MakeDetails("Google Cloud Platform", "listen to events from Google Cloud Platform", feature.Stable), -).MustAddFunction("pubsub", - gcp.NewPubSub, - gcp.PubSubDetails(), -).MustAddFunction("storage", - gcp.NewStorage, - gcp.StorageDetails(), -).Bundle() - -func init() { - feature.MustRegisterBundle(bundle) -} diff --git a/x-pack/functionbeat/provider/gcp/pubsub/pubsub.go b/x-pack/functionbeat/provider/gcp/pubsub/pubsub.go deleted file mode 100644 index 813a0df9cc0..00000000000 --- a/x-pack/functionbeat/provider/gcp/pubsub/pubsub.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package pubsub - -import ( - "context" - "fmt" - - gpubsub "cloud.google.com/go/pubsub" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/x-pack/functionbeat/config" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/beater" - prov "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" - _ "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/include" -) - -func RunPubSub(ctx context.Context, m gpubsub.Message) error { - cfgwarn.Beta("Google Cloud Platform support is in beta") - settings := instance.Settings{ - Name: "functionbeat", - IndexPrefix: "functionbeat", - ConfigOverrides: config.FunctionOverrides, - } - - cfgfile.SetConfigPath("/srv/src/pubsub") - cfgfile.ChangeDefaultCfgfileFlag(settings.Name) - - return instance.Run(settings, initFunctionbeat(ctx, m)) -} - -func initFunctionbeat(ctx context.Context, m gpubsub.Message) func(*beat.Beat, *common.Config) (beat.Beater, error) { - return func(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { - bt, err := beater.New(b, cfg) - if err != nil { - return nil, err - } - - fnbeat, ok := bt.(*beater.Functionbeat) - if !ok { - return nil, fmt.Errorf("not Functionbeat") - } - - fnbeat.Ctx, err = prov.NewPubSubContext(fnbeat.Ctx, ctx, m) - if err != nil { - return nil, err - } - - return fnbeat, nil - } -} diff --git a/x-pack/functionbeat/provider/gcp/storage/storage.go b/x-pack/functionbeat/provider/gcp/storage/storage.go deleted file mode 100644 index 2de829392d2..00000000000 --- a/x-pack/functionbeat/provider/gcp/storage/storage.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package storage - -import ( - "context" - "fmt" - - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/beats/v7/libbeat/cmd/instance" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/x-pack/functionbeat/config" - "github.com/elastic/beats/v7/x-pack/functionbeat/function/beater" - "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/gcp" - _ "github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/include" -) - -func RunCloudStorage(ctx context.Context, e gcp.StorageEvent) error { - cfgwarn.Beta("Google Cloud Platform support is in beta") - settings := instance.Settings{ - Name: "functionbeat", - IndexPrefix: "functionbeat", - ConfigOverrides: config.FunctionOverrides, - } - - cfgfile.SetConfigPath("/srv/src/storage") - cfgfile.ChangeDefaultCfgfileFlag(settings.Name) - - return instance.Run(settings, initFunctionbeat(ctx, e)) -} - -func initFunctionbeat(ctx context.Context, e gcp.StorageEvent) func(*beat.Beat, *common.Config) (beat.Beater, error) { - return func(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { - bt, err := beater.New(b, cfg) - if err != nil { - return nil, err - } - - fnbeat, ok := bt.(*beater.Functionbeat) - if !ok { - return nil, fmt.Errorf("not Functionbeat") - } - - fnbeat.Ctx, err = gcp.NewStorageContext(fnbeat.Ctx, ctx, e) - if err != nil { - return nil, err - } - - return fnbeat, nil - } -} diff --git a/x-pack/functionbeat/scripts/mage/providers.go b/x-pack/functionbeat/scripts/mage/providers.go index 538fcdda071..d77c1da548d 100644 --- a/x-pack/functionbeat/scripts/mage/providers.go +++ b/x-pack/functionbeat/scripts/mage/providers.go @@ -13,7 +13,6 @@ import ( var ( availableProviders = []ProviderDetails{ {Name: "aws", Buildable: true, GOOS: "linux", GOARCH: "amd64"}, - {Name: "gcp", Buildable: false}, } ) diff --git a/x-pack/functionbeat/scripts/mage/update.go b/x-pack/functionbeat/scripts/mage/update.go index 99f1b9a3ce3..468bdafbe0f 100644 --- a/x-pack/functionbeat/scripts/mage/update.go +++ b/x-pack/functionbeat/scripts/mage/update.go @@ -5,13 +5,9 @@ package mage import ( - "os" - "path/filepath" - "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" - "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Update target namespace. @@ -24,7 +20,7 @@ var Aliases = map[string]interface{}{ // All updates all generated content. func (Update) All() { - mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats) + mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs) } // Config generates both the short and reference configs. @@ -50,42 +46,3 @@ func (Update) IncludeFields() error { return devtools.GenerateAllInOneFieldsGo() } - -// VendorBeats collects the vendor folder required to deploy the function for GCP. -func (Update) VendorBeats() error { - for _, f := range []string{"pubsub", "storage"} { - gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor") - err := os.RemoveAll(gcpVendorPath) - if err != nil { - return err - } - - deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f) - if err != nil { - return err - } - - for importPath, location := range deps { - cp := &devtools.CopyTask{ - Source: location, - Dest: filepath.Join(gcpVendorPath, importPath), - Mode: 0600, - DirMode: os.ModeDir | 0750, - Exclude: []string{ - ".*_test.go$", - ".*.yml", - // XXX GCP function metadata lib must be removed to avoid build failures - // GH issue: https://github.com/googleapis/google-cloud-go/issues/1947 - ".*cloud.google.com/go.*/functions/metadata.*", - }, - } - err = cp.Execute() - if err != nil { - return err - } - } - - } - - return nil -}