Cherry-pick #13257 to 7.x: [Metricbeat] [WIP] Add SQL module (#13106)

CHANGELOG.next.asciidoc

@@ -68,6 +68,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/raid metricset now uses /sys/block instead of /proc/mdstat for data. {pull}11613[11613]
 - kubernetes.container.cpu.limit.cores and kubernetes.container.cpu.requests.cores are now floats. {issue}11975[11975]
 - Add statistic option into cloudwatch metricset. If there is no statistic method specified, default is to collect Average, Sum, Maximum, Minimum and SampleCount. {issue}12370[12370] {pull}12840[12840]
+- Add sql module that fetches metrics from a SQL database {pull}13257[13257]
 
 *Packetbeat*
 

NOTICE.txt

@@ -3742,6 +3742,36 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: github.com/jmoiron/sqlx
+Revision: d7d95172beb5a538ff08c50a6e98aee6e7c41f40
+License type (autodetected): MIT
+./x-pack/metricbeat/module/sql/vendor/github.com/jmoiron/sqlx/LICENSE:
+--------------------------------------------------------------------
+ Copyright (c) 2013, Jason Moiron
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------
 Dependency: github.com/joeshaw/multierror
 Revision: 69b34d4ec901851247ae7e77d33909caf9df99ed

metricbeat/docs/fields.asciidoc

@@ -62,6 +62,7 @@ grouped in the following categories:
 * <<exported-fields-prometheus>>
 * <<exported-fields-rabbitmq>>
 * <<exported-fields-redis>>
+* <<exported-fields-sql>>
 * <<exported-fields-statsd>>
 * <<exported-fields-system>>
 * <<exported-fields-tomcat>>
@@ -29316,6 +29317,67 @@ type: long
 
 --
 
+[[exported-fields-sql]]
+== sql fields
+
+sql module
+
+
+
+[float]
+=== sql
+
+`sql` fetches metrics from a SQL database 
+
+
+
+[float]
+=== query
+
+query
+
+
+
+*`sql.query.driver`*::
++
+--
+Driver used to execute the query.
+
+
+type: keyword
+
+--
+
+*`sql.query.query`*::
++
+--
+Query executed to collect metrics.
+
+
+type: keyword
+
+--
+
+*`sql.query.metrics.numeric.*`*::
++
+--
+Numeric metrics collected.
+
+
+type: object
+
+--
+
+*`sql.query.metrics.string.*`*::
++
+--
+Non-numeric values collected.
+
+
+type: object
+
+--
+
 [[exported-fields-statsd]]
 == Statsd fields
 

metricbeat/docs/modules/sql.asciidoc

@@ -0,0 +1,45 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-module-sql]]
+[role="xpack"]
+== sql module
+
+beta[]
+
+This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query.
+
+
+
+
+[float]
+=== Example configuration
+
+The sql module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: sql
+  metricsets:
+    - query
+  period: 10s
+  hosts: ["localhost"]
+
+  driver: "postgres"
+  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
+  sql_query: "select now()"
+
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-sql-query,query>>
+
+include::sql/query.asciidoc[]
+

metricbeat/docs/modules/sql/query.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-sql-query]]
+=== sql query metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/sql/query/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-sql,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/sql/query/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -175,6 +175,8 @@ This file is generated! See scripts/mage/docs_collector.go
 .3+| .3+|  |<<metricbeat-metricset-redis-info,info>>   
 |<<metricbeat-metricset-redis-key,key>>   
 |<<metricbeat-metricset-redis-keyspace,keyspace>>   
+|<<metricbeat-module-sql,sql>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+.1+| .1+|  |<<metricbeat-metricset-sql-query,query>> beta[]  
 |<<metricbeat-module-statsd,Statsd>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-statsd-server,server>> beta[]  
 |<<metricbeat-module-system,System>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
@@ -258,6 +260,7 @@ include::modules/postgresql.asciidoc[]
 include::modules/prometheus.asciidoc[]
 include::modules/rabbitmq.asciidoc[]
 include::modules/redis.asciidoc[]
+include::modules/sql.asciidoc[]
 include::modules/statsd.asciidoc[]
 include::modules/system.asciidoc[]
 include::modules/tomcat.asciidoc[]

x-pack/metricbeat/metricbeat.reference.yml

@@ -905,6 +905,18 @@ metricbeat.modules:
   # Redis AUTH password. Empty by default.
   #password: foobared
 
+#--------------------------------- Sql Module ---------------------------------
+- module: sql
+  metricsets:
+    - query
+  period: 10s
+  hosts: ["localhost"]
+
+  driver: "postgres"
+  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
+  sql_query: "select now()"
+
+
 #-------------------------------- Statsd Module --------------------------------
 - module: statsd
   host: "localhost"

x-pack/metricbeat/module/sql/_meta/config.yml

@@ -0,0 +1,10 @@
+- module: sql
+  metricsets:
+    - query
+  period: 10s
+  hosts: ["localhost"]
+
+  driver: "postgres"
+  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
+  sql_query: "select now()"
+

x-pack/metricbeat/module/sql/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query.
+
+

x-pack/metricbeat/module/sql/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: sql
+  title: "sql"
+  release: beta
+  description: >
+    sql module
+  fields:
+    - name: sql
+      type: group
+      description: >
+        `sql` fetches metrics from a SQL database 
+      fields:

x-pack/metricbeat/module/sql/doc.go

@@ -0,0 +1,6 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// Package sql is a Metricbeat module that collects information from databases using SQL queries.
+package sql

x-pack/metricbeat/module/sql/query/_meta/data.json

@@ -0,0 +1,26 @@
+{
+    "@timestamp":"2016-05-23T08:05:34.853Z",
+    "beat":{
+        "hostname":"beathost",
+        "name":"beathost"
+    },
+    "metricset":{
+        "host":"localhost",
+        "module":"sql",
+        "name":"query",
+        "rtt":44269
+    },
+    "sql":{
+        "metrics":{
+           "numeric":{
+              "mynumericfield":1
+           },
+           "string":{
+              "mystringfield":"abc"
+           }
+        },
+        "driver":"postgres",
+        "query":"select * from mytable"
+     },
+    "type":"metricsets"
+}

x-pack/metricbeat/module/sql/query/_meta/docs.asciidoc

@@ -0,0 +1,4 @@
+The sql `query` metricset collect rows returned by a query. 
+
+Fields names (columns) will be returned as lowercase.
+Values will be returned as numeric or string.

x-pack/metricbeat/module/sql/query/_meta/fields.yml

@@ -0,0 +1,24 @@
+- name: query
+  type: group
+  release: beta
+  description: >
+    query
+  fields:
+    - name: driver
+      type: keyword
+      description: >
+        Driver used to execute the query.
+    - name: query
+      type: keyword
+      description: >
+        Query executed to collect metrics.
+    - name: metrics.numeric.*
+      type: object
+      object_type: double
+      description: >
+        Numeric metrics collected.
+    - name: metrics.string.*
+      type: object
+      object_type: keyword
+      description: >
+        Non-numeric values collected.