Add Nginx stub status to metricbeat (elastic#1642)

* Add Nginx module with stub status metric set
* Return error if Nginx stub status cannot be parsed
* Add Nginx stub status integration test
* Add fields for Nginx and Nginx stub metricset (#2)

metricbeat/docker-compose.yml

@@ -3,11 +3,14 @@ beat:
   links:
     - apache
     - mysql
+    - nginx
     - redis
     - zookeeper
   environment:
     - APACHE_HOST=apache
     - APACHE_PORT=80
+    - NGINX_HOST=nginx
+    - NGINX_PORT=80
     - REDIS_HOST=redis
     - REDIS_PORT=6379
     - MYSQL_DSN=root:test@tcp(mysql:3306)/
@@ -31,6 +34,9 @@ mysql:
   environment:
     - MYSQL_ROOT_PASSWORD=test
 
+nginx:
+  build: ${PWD}/tests/environments/nginx
+
 redis:
   image: redis:3.2.0
 

metricbeat/docker-entrypoint.sh

@@ -19,12 +19,12 @@ waitFor() {
     echo
     echo >&2 '${3} is not available'
     echo >&2 "Address: ${1}:${2}"
-
 }
 
 # Main
 waitFor ${APACHE_HOST} ${APACHE_PORT} Apache
-waitFor ${REDIS_HOST} ${REDIS_PORT} Redis
 waitFor ${MYSQL_HOST} ${MYSQL_PORT} MySQL
+waitFor ${NGINX_HOST} ${NGINX_PORT} Nginx
+waitFor ${REDIS_HOST} ${REDIS_PORT} Redis
 waitFor ${ZOOKEEPER_HOST} ${ZOOKEEPER_PORT} Zookeeper
 exec "$@"

metricbeat/docs/fields.asciidoc

@@ -13,6 +13,7 @@ grouped in the following categories:
 * <<exported-fields-beat>>
 * <<exported-fields-common>>
 * <<exported-fields-mysql>>
+* <<exported-fields-nginx>>
 * <<exported-fields-redis>>
 * <<exported-fields-system>>
 * <<exported-fields-zookeeper>>
@@ -445,6 +446,89 @@ type: integer
 The number of bytes sent to all clients.
 
 
+[[exported-fields-nginx]]
+=== Nginx Status Fields
+
+Nginx server status metrics collected from various modules.
+
+
+
+=== nginx-stubstatus Fields
+
+`nginx-stubstatus` contains the metrics that were scraped from the ngx_http_stub_status_module status page.
+
+
+
+==== nginx-stubstatus.hostname
+
+type: keyword
+
+Nginx hostname
+
+
+==== nginx-stubstatus.active
+
+type: integer
+
+The current number of active client connections including Waiting connections.
+
+
+==== nginx-stubstatus.accepts
+
+type: integer
+
+The total number of accepted client connections.
+
+
+==== nginx-stubstatus.handled
+
+type: integer
+
+The total number of handled client connections.
+
+
+==== nginx-stubstatus.dropped
+
+type: integer
+
+The total number of dropped client connections.
+
+
+==== nginx-stubstatus.requests
+
+type: integer
+
+The total number of client requests.
+
+
+==== nginx-stubstatus.current
+
+type: integer
+
+The current number of client requests.
+
+
+==== nginx-stubstatus.reading
+
+type: integer
+
+The current number of connections where nginx is reading the request header.
+
+
+==== nginx-stubstatus.writing
+
+type: integer
+
+The current number of connections where nginx is writing the response back to the client.
+
+
+==== nginx-stubstatus.waiting
+
+type: integer
+
+The current number of idle client connections waiting for a request.
+
+
 [[exported-fields-redis]]
 === Redis Fields
 

metricbeat/docs/modules/nginx.asciidoc

@@ -0,0 +1,40 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-nginx]]
+== Nginx Module
+
+This module periodically fetches metrics from https://nginx.org/[Nginx] servers.
+
+
+
+=== Example Configuration
+
+The Nginx module supports the standard configuration options which can be found
+here (add link). Below is an example of a configuration option:
+
+[source,yaml]
+----
+metricbeat.modules:
+  #----------------------------- Nginx Module ----------------------------------
+  - module: nginx
+    metricsets: ["stubstatus"]
+    enabled: true
+    period: 1s
+
+    # Nginx hosts
+    hosts: ["http://127.0.0.1/"]
+
+    # Path to server status. Default server-status
+    #server_status_path: "server-status"
+----
+
+=== MetricSets
+
+The following MetricSets are available:
+
+* <<metricbeat-metricset-nginx-stubstatus,stubstatus>>
+
+include::nginx/stubstatus.asciidoc[]
+

metricbeat/docs/modules/nginx/stubstatus.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-nginx-stubstatus]]
+=== Nginx StubStatus MetricSet
+
+The Nginx StubStatus MetricSet collects data from the Nginx
+http://nginx.org/en/docs/http/ngx_http_stub_status_module.html[ngx_http_stub_status] module. It
+scrapes the server status data from the web page generated by ngx_http_stub_status.
+
+
+
+==== Fields
+
+A description of each field in the MetricSet can be found in the
+<<exported-fields-nginx,exported fields>> section
+
+Below is an example document generated by this metricset.
+
+[source,json]
+----
+----

metricbeat/etc/beat.yml

@@ -44,6 +44,18 @@ metricbeat.modules:
   # Password of hosts. Empty by default
   #password: test
 
+#----------------------------- Nginx Module ----------------------------------
+- module: nginx
+  metricsets: ["stubstatus"]
+  enabled: true
+  period: 1s
+
+  # Nginx hosts
+  hosts: ["http://127.0.0.1/"]
+
+  # Path to server status. Default server-status
+  #server_status_path: "server-status"
+
 #------------------------------- Redis Module ---------------------------------
 - module: redis
   metricsets: ["info"]

metricbeat/etc/fields.yml

@@ -249,6 +249,61 @@
                   type: integer
                   description: >
                     The number of bytes sent to all clients.
+- key: nginx
+  title: "Nginx Status"
+  description: >
+    Nginx server status metrics collected from various modules.
+  fields:
+    - name: nginx
+      type: group
+      description: >
+        nginx contains the metrics that were scraped from nginx.
+      fields:
+        - name: nginx-stubstatus
+          type: group
+          description: >
+            `nginx-stubstatus` contains the metrics that were scraped from the ngx_http_stub_status_module status page.
+          fields:
+            - name: hostname
+              type: keyword
+              description: >
+                Nginx hostname
+            - name: active
+              type: integer
+              description: >
+                The current number of active client connections including Waiting connections.
+            - name: accepts
+              type: integer
+              description: >
+                The total number of accepted client connections.
+            - name: handled
+              type: integer
+              description: >
+                The total number of handled client connections.
+            - name: dropped
+              type: integer
+              description: >
+                The total number of dropped client connections.
+            - name: requests
+              type: integer
+              description: >
+                The total number of client requests.
+            - name: current
+              type: integer
+              description: >
+                The current number of client requests.
+            - name: reading
+              type: integer
+              description: >
+                The current number of connections where nginx is reading the request header.
+            - name: writing
+              type: integer
+              description: >
+                The current number of connections where nginx is writing the response back to the client.
+            - name: waiting
+              type: integer
+              description: >
+                The current number of idle client connections waiting for a request.
 - key: redis
   title: "Redis"
   description: >

metricbeat/include/list.go

@@ -13,6 +13,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/apache/status"
 	_ "github.com/elastic/beats/metricbeat/module/mysql"
 	_ "github.com/elastic/beats/metricbeat/module/mysql/status"
+	_ "github.com/elastic/beats/metricbeat/module/nginx"
+	_ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus"
 	_ "github.com/elastic/beats/metricbeat/module/redis"
 	_ "github.com/elastic/beats/metricbeat/module/redis/info"
 	_ "github.com/elastic/beats/metricbeat/module/system"

metricbeat/metricbeat.template-es2x.json

@@ -212,6 +212,42 @@
             }
           }
         },
+        "nginx-stubstatus": {
+          "properties": {
+            "accepts": {
+              "type": "integer"
+            },
+            "active": {
+              "type": "integer"
+            },
+            "current": {
+              "type": "integer"
+            },
+            "dropped": {
+              "type": "integer"
+            },
+            "handled": {
+              "type": "integer"
+            },
+            "hostname": {
+              "ignore_above": 1024,
+              "index": "not_analyzed",
+              "type": "string"
+            },
+            "reading": {
+              "type": "integer"
+            },
+            "requests": {
+              "type": "integer"
+            },
+            "waiting": {
+              "type": "integer"
+            },
+            "writing": {
+              "type": "integer"
+            }
+          }
+        },
         "redis": {
           "properties": {
             "info": {
@@ -644,4 +680,4 @@
     "index.refresh_interval": "5s"
   },
   "template": "metricbeat-*"
-}
+}

metricbeat/metricbeat.template.json

@@ -203,6 +203,41 @@
             }
           }
         },
+        "nginx-stubstatus": {
+          "properties": {
+            "accepts": {
+              "type": "integer"
+            },
+            "active": {
+              "type": "integer"
+            },
+            "current": {
+              "type": "integer"
+            },
+            "dropped": {
+              "type": "integer"
+            },
+            "handled": {
+              "type": "integer"
+            },
+            "hostname": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "reading": {
+              "type": "integer"
+            },
+            "requests": {
+              "type": "integer"
+            },
+            "waiting": {
+              "type": "integer"
+            },
+            "writing": {
+              "type": "integer"
+            }
+          }
+        },
         "redis": {
           "properties": {
             "info": {
@@ -621,4 +656,4 @@
     "index.refresh_interval": "5s"
   },
   "template": "metricbeat-*"
-}
+}