From b28c81117390dadf9489c0c8c38d29d7861e85ad Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 27 May 2020 22:27:31 +0200 Subject: [PATCH] Tag googlepubsub docker image (#18773) (#18784) Add compose targets to x-pack filebeat, adding support for INPUT environment variable in a similar fashion to the MODULE one. Add supported-versions file to googlepubsub so this image can be built with `mage compose:buildSupportedVersions`. Tag the version in docker compose file so image can be build and reused if already exists. Remove googlepubsub from main docker compose file, it is not needed there and images defined there are always rebuilt. (cherry picked from commit cdc7e078e48f60e6d2c33656e751855f4873acf1) --- dev-tools/mage/target/compose/compose.go | 5 +++++ x-pack/filebeat/docker-compose.yml | 6 ------ .../input/googlepubsub/_meta/Dockerfile | 5 +++-- .../googlepubsub/_meta/docker-compose.yml | 7 ------- .../googlepubsub/_meta/supported-versions.yml | 2 ++ .../input/googlepubsub/docker-compose.yml | 11 +++++++++++ .../filebeat/input/googlepubsub/pubsub_test.go | 18 ++++++++++-------- x-pack/filebeat/magefile.go | 2 ++ 8 files changed, 33 insertions(+), 23 deletions(-) delete mode 100644 x-pack/filebeat/input/googlepubsub/_meta/docker-compose.yml create mode 100644 x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml create mode 100644 x-pack/filebeat/input/googlepubsub/docker-compose.yml diff --git a/dev-tools/mage/target/compose/compose.go b/dev-tools/mage/target/compose/compose.go index 8af1666d7526..159e4f92d374 100644 --- a/dev-tools/mage/target/compose/compose.go +++ b/dev-tools/mage/target/compose/compose.go @@ -89,6 +89,11 @@ func findSupportedVersionsFiles() ([]string, error) { return []string{path}, nil } + if input := os.Getenv("INPUT"); len(input) > 0 { + path := filepath.Join("input", input, "_meta/supported-versions.yml") + return []string{path}, nil + } + return devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool { return filepath.Base(path) == "supported-versions.yml" }) diff --git a/x-pack/filebeat/docker-compose.yml b/x-pack/filebeat/docker-compose.yml index 1aff78ea8d9d..af81ccb13fb9 100644 --- a/x-pack/filebeat/docker-compose.yml +++ b/x-pack/filebeat/docker-compose.yml @@ -9,18 +9,12 @@ services: - BEAT_STRICT_PERMS=false - ES_HOST=elasticsearch - ES_PORT=9200 - - PUBSUB_EMULATOR_HOST=googlepubsub:8432 working_dir: /go/src/github.com/elastic/beats/x-pack/filebeat volumes: - ${PWD}/../..:/go/src/github.com/elastic/beats/ - /var/run/docker.sock:/var/run/docker.sock command: make - googlepubsub: - build: input/googlepubsub/_meta - ports: - - 8432 - # This is a proxy used to block beats until all services are healthy. # See: https://github.com/docker/compose/issues/4369 proxy_dep: diff --git a/x-pack/filebeat/input/googlepubsub/_meta/Dockerfile b/x-pack/filebeat/input/googlepubsub/_meta/Dockerfile index 1c47d0aa317f..eea35d423747 100644 --- a/x-pack/filebeat/input/googlepubsub/_meta/Dockerfile +++ b/x-pack/filebeat/input/googlepubsub/_meta/Dockerfile @@ -1,4 +1,5 @@ FROM debian:stretch +ARG SDK_VERSION RUN \ apt-get update \ @@ -20,8 +21,8 @@ RUN \ RUN \ apt-get update \ && apt-get install -y \ - google-cloud-sdk \ - google-cloud-sdk-pubsub-emulator \ + google-cloud-sdk=${SDK_VERSION} \ + google-cloud-sdk-pubsub-emulator=${SDK_VERSION} \ && rm -rf /var/lib/apt/lists/* RUN \ diff --git a/x-pack/filebeat/input/googlepubsub/_meta/docker-compose.yml b/x-pack/filebeat/input/googlepubsub/_meta/docker-compose.yml deleted file mode 100644 index 1b34413967af..000000000000 --- a/x-pack/filebeat/input/googlepubsub/_meta/docker-compose.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: '2.3' - -services: - googlepubsub: - build: . - ports: - - 127.0.0.1:8432:8432 diff --git a/x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml b/x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml new file mode 100644 index 000000000000..ac4ba96e699f --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/_meta/supported-versions.yml @@ -0,0 +1,2 @@ +variants: + - SDK_VERSION: 293.0.0-0 diff --git a/x-pack/filebeat/input/googlepubsub/docker-compose.yml b/x-pack/filebeat/input/googlepubsub/docker-compose.yml new file mode 100644 index 000000000000..6afb855b8a47 --- /dev/null +++ b/x-pack/filebeat/input/googlepubsub/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2.3' + +services: + googlepubsub: + image: docker.elastic.co/integrations-ci/beats-googlepubsub:emulator-${SDK_VERSION:-293.0.0-0}-1 + build: + context: ./_meta + args: + SDK_VERSION: ${SDK_VERSION:-293.0.0-0} + ports: + - 8432 diff --git a/x-pack/filebeat/input/googlepubsub/pubsub_test.go b/x-pack/filebeat/input/googlepubsub/pubsub_test.go index 58d4db9331cc..709cc488fc72 100644 --- a/x-pack/filebeat/input/googlepubsub/pubsub_test.go +++ b/x-pack/filebeat/input/googlepubsub/pubsub_test.go @@ -40,17 +40,19 @@ var once sync.Once func testSetup(t *testing.T) (*pubsub.Client, context.CancelFunc) { t.Helper() - host := os.Getenv("PUBSUB_EMULATOR_HOST") - if host == "" { - t.Skip("PUBSUB_EMULATOR_HOST is not set in environment. You can start " + - "the emulator with \"docker-compose up\" from the _meta directory. " + - "The default address is PUBSUB_EMULATOR_HOST=localhost:8432") - } - + var host string if isInDockerIntegTestEnv() { // We're running inside out integration test environment so // make sure that that googlepubsub container is running. - compose.EnsureUp(t, "googlepubsub") + host = compose.EnsureUp(t, "googlepubsub").Host() + os.Setenv("PUBSUB_EMULATOR_HOST", host) + } else { + host = os.Getenv("PUBSUB_EMULATOR_HOST") + if host == "" { + t.Skip("PUBSUB_EMULATOR_HOST is not set in environment. You can start " + + "the emulator with \"docker-compose up\" from the _meta directory. " + + "The default address is PUBSUB_EMULATOR_HOST=localhost:8432") + } } once.Do(func() { diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go index c1688f2bcc15..691977b89eb9 100644 --- a/x-pack/filebeat/magefile.go +++ b/x-pack/filebeat/magefile.go @@ -21,6 +21,8 @@ import ( // mage:import generate _ "github.com/elastic/beats/v7/filebeat/scripts/mage/generate" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/test"