telegraf.conf

[agent]
  interval = "1s"
  round_interval = true
  metric_batch_size = 1000
  metric_buffer_limit = 10000
  collection_jitter = "0s"
  flush_interval = "1s"
  flush_jitter = "0s"
  precision = ""
  debug = false
  quiet = false
  logfile = ""
  hostname = ""
  omit_hostname = false

###############################################################################
#                            OUTPUT PLUGINS                                   #
###############################################################################

[[outputs.remp_elasticsearch]]
  alias = "pageviews"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "pageviews"
  namepass = ["pageviews"]
  manage_template = false
  id_field = "remp_pageview_id"
  tagexclude = ["host"]

[[outputs.remp_elasticsearch]]
  alias = "timespent"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "pageviews_time_spent"
  namepass = ["pageviews_time_spent"]
  manage_template = false
  id_field = "remp_pageview_id"
  updated_fields = ["timespent"]
  tagexclude = ["host"]
  field_whitelist = ["token", "subscriber", "signed_in", "locked", "timespent", "unload"]

[[outputs.remp_elasticsearch]]
  alias = "progress"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "pageviews_progress"
  namepass = ["pageviews_progress"]
  manage_template = false
  id_field = "remp_pageview_id"
  updated_fields = ["page_progress", "article_progress"]
  field_whitelist = ["token", "subscriber", "signed_in", "locked", "page_progress", "article_progress", "unload"]

[[outputs.remp_elasticsearch]]
  alias = "events"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "events"
  namepass = ["events_v2"]
  manage_template = false
  id_field = "remp_event_id"
  tagexclude = ["host"]

[[outputs.remp_elasticsearch]]
  alias = "commerce"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "commerce"
  namepass = ["commerce"]
  manage_template = false
  id_field = "remp_commerce_id"
  tagexclude = ["host"]

[[outputs.remp_elasticsearch]]
  alias = "concurrents"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "concurrents_by_browser"
  # if you to track timespent, keep current setting as is so the concurrents can be based on more recent data
  namepass = ["pageviews_time_spent"]
  # if you don't plan to track timespent, you can make concurrents_by_browser based on regular pageviews; the data will be a bit less accurate
  # namepass = ["pageviews"]
  manage_template = false
  id_field = "browser_id"
  updated_fields = ["time", "article_id", "token", "derived_referer_host_with_path", "derived_referer_medium", "url", "canonical_url", "derived_ua_device"]
  field_whitelist = ["time", "article_id", "token", "derived_referer_host_with_path", "derived_referer_medium", "url", "canonical_url", "derived_ua_device"]

[[outputs.remp_elasticsearch]]
  alias = "entities"
  urls = ["http://elasticsearch:9200"] # required
  timeout = "5s"
  index_name = "entities"
  namepass = ["entities"]
  manage_template = false
  id_field = "remp_entity_id"
  tagexclude = ["host"]

###############################################################################
#                            SERVICE INPUT PLUGINS                            #
###############################################################################

[[inputs.kafka_consumer]]
  topics = ["beam_events"]
  brokers = ["kafka:9092"]
  consumer_group = "beam_consumers"
  offset = "oldest"
  data_format = "influx"

## Enable if using Pub/Sub
#[[inputs.cloud_pubsub]]
#  ## Required. Name of Google Cloud Platform (GCP) Project that owns
#  ## the given PubSub subscription.
#  project = "YOUR_PROJECT_ID"
#
#  ## Required. Name of PubSub subscription to ingest metrics from.
#  subscription = "PUB_SUB_SUBSCRIPTION_NAME"
#
#  data_format = "influx"

#  ## Optional. Filepath for GCP credentials JSON file to authorize calls to
#  ## PubSub APIs. If not set explicitly, Telegraf will attempt to use
#  ## Application Default Credentials, which is preferred.
#  credentials_file = "/path/to/keys.json"