From 86004dd4ba40b40e3e88344e7e4c0e47670886d7 Mon Sep 17 00:00:00 2001 From: Jacky Wu Date: Wed, 22 Sep 2021 10:58:26 +0800 Subject: [PATCH] feat: add huaweicloud metadata support. (#27607) Signed-off-by: colstuwjx Co-authored-by: Jaime Soriano Pastor --- CHANGELOG.next.asciidoc | 1 + .../docs/add_cloud_metadata.asciidoc | 16 +++ .../provider_huawei_cloud.go | 80 +++++++++++++++ .../provider_huawei_cloud_test.go | 97 +++++++++++++++++++ .../add_cloud_metadata/providers.go | 1 + 5 files changed, 195 insertions(+) create mode 100644 libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go create mode 100644 libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 19441555726..252ea3f7b3e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -532,6 +532,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Allow non-padded base64 data to be decoded by decode_base64_field {pull}27311[27311], {issue}27021[27021] - The Kafka support library Sarama has been updated to 1.29.1. {pull}27717[27717] - Kafka is now supported up to version 2.8.0. {pull}27720[27720] +- Add Huawei Cloud provider to add_cloud_metadata. {pull}27607[27607] - Add default seccomp policy for linux arm64. {pull}27955[27955] *Auditbeat* diff --git a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc index 44497f31539..a302aa6cd0f 100644 --- a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc +++ b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc @@ -16,6 +16,7 @@ The following cloud providers are supported: - Google Compute Engine (GCE) - https://www.qcloud.com/?lang=en[Tencent Cloud] (QCloud) - Alibaba Cloud (ECS) +- Huawei Cloud (ECS) - Azure Virtual Machine - Openstack Nova @@ -54,6 +55,7 @@ List of names the `providers` setting supports: - "openstack", or "nova" for Openstack Nova (enabled by default). - "openstack-ssl", or "nova-ssl" for Openstack Nova when SSL metadata APIs are enabled (enabled by default). - "tencent", or "qcloud" for Tencent Cloud (disabled by default). +- "huawei" for Huawei Cloud (enabled by default). The third optional configuration setting is `overwrite`. When `overwrite` is `true`, `add_cloud_metadata` overwrites existing `cloud.*` fields (`false` by @@ -124,6 +126,20 @@ _Tencent Cloud_ } ------------------------------------------------------------------------------- +_Huawei Cloud_ + +[source,json] +------------------------------------------------------------------------------- +{ + "cloud": { + "availability_zone": "cn-east-2b", + "instance.id": "37da9890-8289-4c58-ba34-a8271c4a8216", + "provider": "huawei", + "region": "cn-east-2" + } +} +------------------------------------------------------------------------------- + _Alibaba Cloud_ This metadata is only available when VPC is selected as the network type of the diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go new file mode 100644 index 00000000000..01a87b19c15 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "encoding/json" + + "github.com/elastic/beats/v7/libbeat/common" +) + +type hwMeta struct { + ImageName string `json:"image_name"` + VpcID string `json:"vpc_id"` +} + +type hwMetadata struct { + UUID string `json:"uuid"` + AvailabilityZone string `json:"availability_zone"` + RegionID string `json:"region_id"` + Meta *hwMeta `json:"meta"` + ProjectID string `json:"project_id"` + Name string `json:"name"` +} + +// Huawei Cloud Metadata Service +// Document https://support.huaweicloud.com/usermanual-ecs/ecs_03_0166.html +var huaweiMetadataFetcher = provider{ + Name: "huawei-cloud", + + Local: true, + + Create: func(_ string, c *common.Config) (metadataFetcher, error) { + metadataHost := "169.254.169.254" + huaweiCloudMetadataJSONURI := "/openstack/latest/meta_data.json" + + huaweiCloudSchema := func(m map[string]interface{}) common.MapStr { + m["service"] = common.MapStr{ + "name": "ECS", + } + return common.MapStr{"cloud": m} + } + + urls, err := getMetadataURLs(c, metadataHost, []string{ + huaweiCloudMetadataJSONURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + data := new(hwMetadata) + err := json.Unmarshal(all, data) + if err != nil { + return err + } + result.metadata.Put("instance.id", data.UUID) + result.metadata.Put("region", data.RegionID) + result.metadata.Put("availability_zone", data.AvailabilityZone) + return nil + }, + } + fetcher := &httpMetadataFetcher{"huawei", nil, responseHandlers, huaweiCloudSchema} + return fetcher, nil + }, +} diff --git a/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go new file mode 100644 index 00000000000..b6d9c20bb6d --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/provider_huawei_cloud_test.go @@ -0,0 +1,97 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func initHuaweiCloudTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/openstack/latest/meta_data.json" { + w.Write([]byte(`{ + "random_seed": "CWIZtYK4y5pzMtShTtCKx16qB1DsA/2kL0US4u1fHxedODNr7gos4RgdE/z9eHucnltnlJfDY1remfGL60yzTsvEIWPdECOpPaJm1edIYQaUvQzdeQwKcOQAHjUP5wLQzGA3j3Pw10p7u+M7glHEwNRoEY1WsbVYwzyOOkBnqb+MJ1aOhiRnfNtHOxjLNBSDvjHaQZzoHL+1YNAxDYFezE83nE2m3ciVwZO7xWpdKDQ+W5hYBUsYAWODRMOYqIR/5ZLsfAfxE2DhK+NvuMyJ5yjO+ObQf0DN5nRUSrM5ajs84UVMr9ylJuT78ckh83CLSttsjzXJ+sr07ZFsB6/6NABzziFL7Xn8z/mEBVmFXBiBgg7KcWSoH756w42VSdUezwTy9lW0spRmdvNBKV/PzrYyy0FMiGXXZwMOCyBD05CBRJlsPorwxZLlfRVmNvsTuMYB8TG3UUbFhoR8Bd5en+EC3ncH3QIUDWn0oVg28BVjWe5rADVQLX1h83ti6GD08YUGaxoNPXnJLZfiaucSacby2mG31xysxd8Tg0qPRq7744a1HPVryuauWR9pF0+qDmtskhenxK0FR+TQ4w0fRxTigteBsXx1pQu0iz+B8rP68uokU2faCC2IMHY2Tf9RPCe6Eef0/DdQhBft88PuJLwq52o/0qZ/n9HFL6LdgCU=", + "uuid": "37da9890-8289-4c58-ba34-a8271c4a8216", + "availability_zone": "cn-east-2b", + "enterprise_project_id": "0", + "launch_index": 0, + "instance_type": "c3.large.2", + "meta": { + "os_bit": "64", + "image_name": "CentOS 7.4", + "vpc_id": "6dad7f50-db1d-4cce-b095-d27bc837d4bb" + }, + "region_id": "cn-east-2", + "project_id": "c09b8baf28b845a9b53ed37575cfd61f", + "name": "hwdev-test-1" + }`)) + return + } + + http.Error(w, "not found", http.StatusNotFound) + })) +} + +func TestRetrieveHuaweiCloudMetadata(t *testing.T) { + logp.TestingSetup() + + server := initHuaweiCloudTestServer() + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "providers": []string{"huawei"}, + "host": server.Listener.Addr().String(), + }) + + if err != nil { + t.Fatal(err) + } + + p, err := New(config) + if err != nil { + t.Fatal(err) + } + + actual, err := p.Run(&beat.Event{Fields: common.MapStr{}}) + if err != nil { + t.Fatal(err) + } + + expected := common.MapStr{ + "cloud": common.MapStr{ + "provider": "huawei", + "instance": common.MapStr{ + "id": "37da9890-8289-4c58-ba34-a8271c4a8216", + }, + "region": "cn-east-2", + "availability_zone": "cn-east-2b", + "service": common.MapStr{ + "name": "ECS", + }, + }, + } + assert.Equal(t, expected, actual.Fields) +} diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index ea03b64258a..50346bcb126 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -64,6 +64,7 @@ var cloudMetaProviders = map[string]provider{ "nova-ssl": openstackNovaSSLMetadataFetcher, "qcloud": qcloudMetadataFetcher, "tencent": qcloudMetadataFetcher, + "huawei": huaweiMetadataFetcher, } func selectProviders(configList providerList, providers map[string]provider) map[string]provider {