Rename google-pubsub Filebeat input to gcp-pubsub (#22213) (#22499)

* Rename google-pubsub Filebeat input to gcp-pubsub * change to gcp-pubsub input name in elastic-agent * add warning log message for deprecated old input name (cherry picked from commit 42428a8)
elastic · Nov 10, 2020 · 59455a9 · 59455a9
1 parent 8c89921
commit 59455a9
Show file tree

Hide file tree

Showing 21 changed files with 51 additions and 32 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -496,6 +496,7 @@ from being added to events by default. {pull}18159[18159]
 - Add max_number_of_messages config into s3 input. {pull}21993[21993]
 - Update Okta documentation for new stateful restarts. {pull}22091[22091]
 - Add SSL option to checkpoint module {pull}19560[19560]
+- Rename google-pubsub input to gcp-pubsub. {pull}22213[22213]
 - Copy tag names from MISP data into events. {pull}21664[21664]
 - Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291]
 - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]

diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc
@@ -66,7 +66,7 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-cloudfoundry>>
 * <<{beatname_lc}-input-container>>
 * <<{beatname_lc}-input-docker>>
-* <<{beatname_lc}-input-google-pubsub>>
+* <<{beatname_lc}-input-gcp-pubsub>>
 * <<{beatname_lc}-input-http_endpoint>>
 * <<{beatname_lc}-input-httpjson>>
 * <<{beatname_lc}-input-kafka>>
@@ -96,7 +96,7 @@ include::inputs/input-docker.asciidoc[]
 
 include::inputs/input-filestream.asciidoc[]
 
-include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[]
+include::../../x-pack/filebeat/docs/inputs/input-gcp-pubsub.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc[]
 

diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go
diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml
@@ -64,7 +64,7 @@ rules:
     - cloudfoundry
     - container
     - docker
-    - google-pubsub
+    - gcp-pubsub
     - http_endpoint
     - httpjson
     - kafka

diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl
@@ -30,7 +30,7 @@
 
 #---------------------------- Google Cloud Pub/Sub Input -----------------------
 # Input for reading messages from a Google Cloud Pub/Sub topic subscription.
-- type: google-pubsub
+- type: gcp-pubsub
   enabled: false
 
   # Google Cloud project ID. Required.

diff --git a/.../docs/inputs/input-google-pubsub.asciidoc → ...eat/docs/inputs/input-gcp-pubsub.asciidoc b/.../docs/inputs/input-google-pubsub.asciidoc → ...eat/docs/inputs/input-gcp-pubsub.asciidoc
@@ -1,15 +1,15 @@
 [role="xpack"]
 
-:type: google-pubsub
+:type: gcp-pubsub
 
 [id="{beatname_lc}-input-{type}"]
-=== Google Cloud Pub/Sub input
+=== GCP Pub/Sub input
 
 ++++
-<titleabbrev>Google Pub/Sub</titleabbrev>
+<titleabbrev>GCP Pub/Sub</titleabbrev>
 ++++
 
-Use the `google-pubsub` input to read messages from a Google Cloud Pub/Sub topic
+Use the `gcp-pubsub` input to read messages from a Google Cloud Pub/Sub topic
 subscription.
 
 This input can, for example, be used to receive Stackdriver logs that have been
@@ -23,7 +23,7 @@ Example configuration:
 ["source","yaml",subs="attributes"]
 ----
 {beatname_lc}.inputs:
-- type: google-pubsub
+- type: gcp-pubsub
   project_id: my-gcp-project-id
   topic: vpc-firewall-logs-topic
   subscription.name: filebeat-vpc-firewall-logs-sub
@@ -33,7 +33,7 @@ Example configuration:
 
 ==== Configuration options
 
-The `google-pubsub` input supports the following configuration options plus the
+The `gcp-pubsub` input supports the following configuration options plus the
 <<{beatname_lc}-input-{type}-common-options>> described later.
 
 [float]

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -2388,7 +2388,7 @@ filebeat.inputs:
 
 #---------------------------- Google Cloud Pub/Sub Input -----------------------
 # Input for reading messages from a Google Cloud Pub/Sub topic subscription.
-- type: google-pubsub
+- type: gcp-pubsub
   enabled: false
 
   # Google Cloud project ID. Required.

diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go
diff --git a/...ebeat/input/googlepubsub/_meta/Dockerfile → ...filebeat/input/gcppubsub/_meta/Dockerfile b/...ebeat/input/googlepubsub/_meta/Dockerfile → ...filebeat/input/gcppubsub/_meta/Dockerfile
diff --git a/...googlepubsub/_meta/supported-versions.yml → ...ut/gcppubsub/_meta/supported-versions.yml b/...googlepubsub/_meta/supported-versions.yml → ...ut/gcppubsub/_meta/supported-versions.yml
diff --git a/x-pack/filebeat/input/googlepubsub/config.go → x-pack/filebeat/input/gcppubsub/config.go b/x-pack/filebeat/input/googlepubsub/config.go → x-pack/filebeat/input/gcppubsub/config.go
@@ -2,18 +2,21 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
 	"context"
 	"fmt"
 	"os"
 
+	"github.com/elastic/beats/v7/filebeat/harvester"
+
 	"cloud.google.com/go/pubsub"
 	"golang.org/x/oauth2/google"
 )
 
 type config struct {
+	harvester.ForwarderConfig `config:",inline"`
 	// Google Cloud project name.
 	ProjectID string `config:"project_id" validate:"required"`
 
@@ -62,6 +65,9 @@ func (c *config) Validate() error {
 
 func defaultConfig() config {
 	var c config
+	c.ForwarderConfig = harvester.ForwarderConfig{
+		Type: "gcp-pubsub",
+	}
 	c.Subscription.NumGoroutines = 1
 	c.Subscription.MaxOutstandingMessages = 1000
 	c.Subscription.Create = true

diff --git a/...ilebeat/input/googlepubsub/config_test.go → ...k/filebeat/input/gcppubsub/config_test.go b/...ilebeat/input/googlepubsub/config_test.go → ...k/filebeat/input/gcppubsub/config_test.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
 	"os"

diff --git a/...eat/input/googlepubsub/docker-compose.yml → ...lebeat/input/gcppubsub/docker-compose.yml b/...eat/input/googlepubsub/docker-compose.yml → ...lebeat/input/gcppubsub/docker-compose.yml
diff --git a/x-pack/filebeat/input/googlepubsub/input.go → x-pack/filebeat/input/gcppubsub/input.go b/x-pack/filebeat/input/googlepubsub/input.go → x-pack/filebeat/input/gcppubsub/input.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
 	"context"
@@ -26,14 +26,20 @@ import (
 )
 
 const (
-	inputName = "google-pubsub"
+	inputName    = "gcp-pubsub"
+	oldInputName = "google-pubsub"
 )
 
 func init() {
 	err := input.Register(inputName, NewInput)
 	if err != nil {
 		panic(errors.Wrapf(err, "failed to register %v input", inputName))
 	}
+
+	err = input.Register(oldInputName, NewInput)
+	if err != nil {
+		panic(errors.Wrapf(err, "failed to register %v input", oldInputName))
+	}
 }
 
 type pubsubInput struct {
@@ -64,6 +70,15 @@ func NewInput(
 		return nil, err
 	}
 
+	logger := logp.NewLogger("gcp.pubsub").With(
+		"pubsub_project", conf.ProjectID,
+		"pubsub_topic", conf.Topic,
+		"pubsub_subscription", conf.Subscription)
+
+	if conf.Type == oldInputName {
+		logger.Warnf("%s input name is deprecated, please use %s instead", oldInputName, inputName)
+	}
+
 	// Wrap input.Context's Done channel with a context.Context. This goroutine
 	// stops with the parent closes the Done channel.
 	inputCtx, cancelInputCtx := context.WithCancel(context.Background())
@@ -80,11 +95,8 @@ func NewInput(
 	workerCtx, workerCancel := context.WithCancel(inputCtx)
 
 	in := &pubsubInput{
-		config: conf,
-		log: logp.NewLogger("google.pubsub").With(
-			"pubsub_project", conf.ProjectID,
-			"pubsub_topic", conf.Topic,
-			"pubsub_subscription", conf.Subscription),
+		config:       conf,
+		log:          logger,
 		inputCtx:     inputCtx,
 		workerCtx:    workerCtx,
 		workerCancel: workerCancel,
@@ -109,7 +121,7 @@ func NewInput(
 	if err != nil {
 		return nil, err
 	}
-	in.log.Info("Initialized Google Pub/Sub input.")
+	in.log.Info("Initialized GCP Pub/Sub input.")
 	return in, nil
 }
 

diff --git a/...ilebeat/input/googlepubsub/pubsub_test.go → ...k/filebeat/input/gcppubsub/pubsub_test.go b/...ilebeat/input/googlepubsub/pubsub_test.go → ...k/filebeat/input/gcppubsub/pubsub_test.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package googlepubsub
+package gcppubsub
 
 import (
 	"context"

diff --git a/...eat/input/googlepubsub/testdata/fake.json → ...lebeat/input/gcppubsub/testdata/fake.json b/...eat/input/googlepubsub/testdata/fake.json → ...lebeat/input/gcppubsub/testdata/fake.json
diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}

diff --git a/x-pack/filebeat/module/googlecloud/audit/manifest.yml b/x-pack/filebeat/module/googlecloud/audit/manifest.yml
@@ -2,7 +2,7 @@ module_version: "1.0"
 
 var:
   - name: input
-    default: google-pubsub
+    default: gcp-pubsub
   - name: project_id
     default: SET_PROJECT_NAME
   - name: topic

diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}

diff --git a/x-pack/filebeat/module/googlecloud/firewall/manifest.yml b/x-pack/filebeat/module/googlecloud/firewall/manifest.yml
@@ -2,7 +2,7 @@ module_version: "1.0"
 
 var:
   - name: input
-    default: google-pubsub
+    default: gcp-pubsub
   - name: project_id
     default: SET_PROJECT_NAME
   - name: topic

diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml
@@ -1,6 +1,6 @@
-{{ if eq .input "google-pubsub" }}
+{{ if eq .input "gcp-pubsub" }}
 
-type: google-pubsub
+type: gcp-pubsub
 project_id: {{ .project_id }}
 topic: {{ .topic }}
 subscription.name: {{ .subscription_name }}