From aaadaea228aaa277a1fc244af38fa91c9be7279d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 20 Jan 2020 20:12:31 +0100 Subject: [PATCH 1/9] Add tests for SQL module --- .../metricbeat/module/sql/docker-compose.yml | 12 +++ .../sql/query/query_integration_test.go | 75 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 x-pack/metricbeat/module/sql/docker-compose.yml create mode 100644 x-pack/metricbeat/module/sql/query/query_integration_test.go diff --git a/x-pack/metricbeat/module/sql/docker-compose.yml b/x-pack/metricbeat/module/sql/docker-compose.yml new file mode 100644 index 000000000000..a053c322d304 --- /dev/null +++ b/x-pack/metricbeat/module/sql/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2.3' + +services: + mysql: + extends: + file: ../../../../metricbeat/module/mysql/docker-compose.yml + service: mysql + + postgresql: + extends: + file: ../../../../metricbeat/module/postgresql/docker-compose.yml + service: postgresql diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go new file mode 100644 index 000000000000..597dc2d4cc00 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -0,0 +1,75 @@ +// +build integration + +package query + +import ( + "fmt" + "net" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/mysql" + "github.com/elastic/beats/metricbeat/module/postgresql" + + // Drivers + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" +) + +type testFetchConfig struct { + Driver string + Query string + Host string + Datasource string +} + +func TestFetchMySQL(t *testing.T) { + service := compose.EnsureUp(t, "mysql") + testFetch(t, testFetchConfig{ + Driver: "mysql", + Query: "select now()", + Host: service.Host(), + Datasource: mysql.GetMySQLEnvDSN(service.Host()), + }) +} + +func TestFetchPostgreSQL(t *testing.T) { + service := compose.EnsureUp(t, "postgresql") + host, port, err := net.SplitHostPort(service.Host()) + require.NoError(t, err) + + user := postgresql.GetEnvUsername() + password := postgresql.GetEnvPassword() + + testFetch(t, testFetchConfig{ + Driver: "postgres", + Query: "select now()", + Host: service.Host(), + Datasource: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), + }) +} + +func TestData(t *testing.T) { +} + +func testFetch(t *testing.T, cfg testFetchConfig) { + m := mbtest.NewFetcher(t, getConfig(cfg)) + events, errs := m.FetchEvents() + require.Empty(t, errs) + require.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) +} + +func getConfig(cfg testFetchConfig) map[string]interface{} { + return map[string]interface{}{ + "module": "sql", + "metricsets": []string{"query"}, + "hosts": []string{cfg.Host}, + "driver": cfg.Driver, + "sql_query": cfg.Query, + "datasource": cfg.Datasource, + } +} From 3c8384f5db61b09d13c022c794547af6cf68fcad Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 20 Jan 2020 20:32:32 +0100 Subject: [PATCH 2/9] Remove datasource and add data.json --- metricbeat/docs/modules/sql.asciidoc | 5 +- x-pack/metricbeat/metricbeat.reference.yml | 3 +- x-pack/metricbeat/module/sql/_meta/config.yml | 3 +- .../metricbeat/module/sql/_meta/docs.asciidoc | 2 +- .../module/sql/query/_meta/data.json | 46 ++++++++++--------- x-pack/metricbeat/module/sql/query/query.go | 20 ++++---- .../sql/query/query_integration_test.go | 44 ++++++++++-------- x-pack/metricbeat/modules.d/sql.yml.disabled | 3 +- 8 files changed, 66 insertions(+), 60 deletions(-) diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 30e5baf89da6..c3f3d412ea77 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -8,7 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go beta[] -This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query. +This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. @@ -26,10 +26,9 @@ metricbeat.modules: metricsets: - query period: 10s - hosts: ["localhost"] + hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"] driver: "postgres" - datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable" sql_query: "select now()" ---- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b33748020372..6ae445573733 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -965,10 +965,9 @@ metricbeat.modules: metricsets: - query period: 10s - hosts: ["localhost"] + hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"] driver: "postgres" - datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable" sql_query: "select now()" diff --git a/x-pack/metricbeat/module/sql/_meta/config.yml b/x-pack/metricbeat/module/sql/_meta/config.yml index 92b5f3882210..5c6a419e77ba 100644 --- a/x-pack/metricbeat/module/sql/_meta/config.yml +++ b/x-pack/metricbeat/module/sql/_meta/config.yml @@ -2,9 +2,8 @@ metricsets: - query period: 10s - hosts: ["localhost"] + hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"] driver: "postgres" - datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable" sql_query: "select now()" diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index d7bb818a58b8..f22edf1fa200 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -1,3 +1,3 @@ -This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query. +This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. diff --git a/x-pack/metricbeat/module/sql/query/_meta/data.json b/x-pack/metricbeat/module/sql/query/_meta/data.json index 1a92415be34c..21531d552dc3 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data.json @@ -1,26 +1,30 @@ { - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" }, - "metricset":{ - "host":"localhost", - "module":"sql", - "name":"query", - "rtt":44269 + "metricset": { + "name": "query", + "period": 10000 }, - "sql":{ - "metrics":{ - "numeric":{ - "mynumericfield":1 - }, - "string":{ - "mystringfield":"abc" - } + "service": { + "address": "root:test@tcp(172.22.0.3:3306)/", + "type": "sql" + }, + "sql": { + "driver": "mysql", + "metrics": { + "numeric": { + "table_rows": 6 + }, + "string": { + "engine": "InnoDB", + "table_name": "sys_config", + "table_schema": "sys" + } }, - "driver":"postgres", - "query":"select * from mytable" - }, - "type":"metricsets" + "query": "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows \u003e 0;" + } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index 884df1b9df91..b9be69a4d2eb 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -15,6 +15,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" "github.com/jmoiron/sqlx" ) @@ -24,7 +25,9 @@ import ( // the MetricSet for each host defined in the module's configuration. After the // MetricSet has been created then Fetch will begin to be called periodically. func init() { - mb.Registry.MustAddMetricSet("sql", "query", New) + mb.Registry.MustAddMetricSet("sql", "query", New, + mb.WithHostParser(parse.PassThruHostParser), + ) } // MetricSet holds any configuration or state information. It must implement @@ -33,9 +36,10 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - Driver string - Datasource string - Query string + Driver string + Query string + + db *sqlx.DB } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -44,9 +48,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Beta("The sql query metricset is beta.") config := struct { - Driver string `config:"driver"` - Datasource string `config:"datasource"` - Query string `config:"sql_query"` + Driver string `config:"driver"` + Query string `config:"sql_query"` }{} if err := base.Module().UnpackConfig(&config); err != nil { @@ -56,7 +59,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, Driver: config.Driver, - Datasource: config.Datasource, Query: config.Query, }, nil } @@ -65,7 +67,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { - db, err := sqlx.Open(m.Driver, m.Datasource) + db, err := sqlx.Open(m.Driver, m.HostData().URI) if err != nil { return errors.Wrap(err, "error opening connection") } diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 597dc2d4cc00..6d6ede920ad4 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -9,30 +9,28 @@ import ( "github.com/stretchr/testify/require" + // Drivers + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/mysql" "github.com/elastic/beats/metricbeat/module/postgresql" - - // Drivers - _ "github.com/go-sql-driver/mysql" - _ "github.com/lib/pq" ) type testFetchConfig struct { - Driver string - Query string - Host string - Datasource string + Driver string + Query string + Host string } func TestFetchMySQL(t *testing.T) { service := compose.EnsureUp(t, "mysql") testFetch(t, testFetchConfig{ - Driver: "mysql", - Query: "select now()", - Host: service.Host(), - Datasource: mysql.GetMySQLEnvDSN(service.Host()), + Driver: "mysql", + Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", + Host: mysql.GetMySQLEnvDSN(service.Host()), }) } @@ -45,16 +43,12 @@ func TestFetchPostgreSQL(t *testing.T) { password := postgresql.GetEnvPassword() testFetch(t, testFetchConfig{ - Driver: "postgres", - Query: "select now()", - Host: service.Host(), - Datasource: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), + Driver: "postgres", + Query: "select now()", + Host: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), }) } -func TestData(t *testing.T) { -} - func testFetch(t *testing.T, cfg testFetchConfig) { m := mbtest.NewFetcher(t, getConfig(cfg)) events, errs := m.FetchEvents() @@ -63,6 +57,17 @@ func testFetch(t *testing.T, cfg testFetchConfig) { t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) } +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "mysql") + cfg := getConfig(testFetchConfig{ + Driver: "mysql", + Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", + Host: mysql.GetMySQLEnvDSN(service.Host()), + }) + m := mbtest.NewFetcher(t, cfg) + m.WriteEvents(t, "") +} + func getConfig(cfg testFetchConfig) map[string]interface{} { return map[string]interface{}{ "module": "sql", @@ -70,6 +75,5 @@ func getConfig(cfg testFetchConfig) map[string]interface{} { "hosts": []string{cfg.Host}, "driver": cfg.Driver, "sql_query": cfg.Query, - "datasource": cfg.Datasource, } } diff --git a/x-pack/metricbeat/modules.d/sql.yml.disabled b/x-pack/metricbeat/modules.d/sql.yml.disabled index ab839d64f490..31f15547123a 100644 --- a/x-pack/metricbeat/modules.d/sql.yml.disabled +++ b/x-pack/metricbeat/modules.d/sql.yml.disabled @@ -5,9 +5,8 @@ metricsets: - query period: 10s - hosts: ["localhost"] + hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"] driver: "postgres" - datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable" sql_query: "select now()" From 38e8f65fe8bf15d42307593b8be76f3fc2d9c4d7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 20 Jan 2020 20:52:12 +0100 Subject: [PATCH 3/9] Add example data for postgresql too --- .../module/sql/query/_meta/data_postgres.json | 45 +++++++++++++++++++ .../sql/query/query_integration_test.go | 38 ++++++++++------ 2 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 x-pack/metricbeat/module/sql/query/_meta/data_postgres.json diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json new file mode 100644 index 000000000000..3bfd814bc732 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json @@ -0,0 +1,45 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "user=postgres password=postgres sslmode=disable host=172.22.0.2 port=5432", + "type": "sql" + }, + "sql": { + "driver": "postgres", + "metrics": { + "numeric": { + "blk_read_time": 0, + "blk_write_time": 0, + "blks_hit": 4251, + "blks_read": 103, + "conflicts": 0, + "datid": 12379, + "deadlocks": 0, + "numbackends": 1, + "temp_bytes": 0, + "temp_files": 0, + "tup_deleted": 0, + "tup_fetched": 2847, + "tup_inserted": 0, + "tup_returned": 4877, + "tup_updated": 0, + "xact_commit": 49, + "xact_rollback": 0 + }, + "string": { + "datname": "postgres", + "stats_reset": "2020-01-20 19:48:28.217" + } + }, + "query": "select * from pg_stat_database" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 6d6ede920ad4..27738142b8a3 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -25,16 +25,24 @@ type testFetchConfig struct { Host string } -func TestFetchMySQL(t *testing.T) { +func TestMySQL(t *testing.T) { service := compose.EnsureUp(t, "mysql") - testFetch(t, testFetchConfig{ + config := testFetchConfig{ Driver: "mysql", Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", Host: mysql.GetMySQLEnvDSN(service.Host()), + } + + t.Run("fetch", func(t *testing.T) { + testFetch(t, config) + }) + + t.Run("data", func(t *testing.T) { + testData(t, config, "") }) } -func TestFetchPostgreSQL(t *testing.T) { +func TestPostgreSQL(t *testing.T) { service := compose.EnsureUp(t, "postgresql") host, port, err := net.SplitHostPort(service.Host()) require.NoError(t, err) @@ -42,10 +50,18 @@ func TestFetchPostgreSQL(t *testing.T) { user := postgresql.GetEnvUsername() password := postgresql.GetEnvPassword() - testFetch(t, testFetchConfig{ + config := testFetchConfig{ Driver: "postgres", - Query: "select now()", + Query: "select * from pg_stat_database", Host: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), + } + + t.Run("fetch", func(t *testing.T) { + testFetch(t, config) + }) + + t.Run("data", func(t *testing.T) { + testData(t, config, "./_meta/data_postgres.json") }) } @@ -57,15 +73,9 @@ func testFetch(t *testing.T, cfg testFetchConfig) { t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) } -func TestData(t *testing.T) { - service := compose.EnsureUp(t, "mysql") - cfg := getConfig(testFetchConfig{ - Driver: "mysql", - Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", - Host: mysql.GetMySQLEnvDSN(service.Host()), - }) - m := mbtest.NewFetcher(t, cfg) - m.WriteEvents(t, "") +func testData(t *testing.T, cfg testFetchConfig, postfix string) { + m := mbtest.NewFetcher(t, getConfig(cfg)) + m.WriteEvents(t, postfix) } func getConfig(cfg testFetchConfig) map[string]interface{} { From e260032a9b8dd600a98057dbe6cd9ae4a1c39830 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 20 Jan 2020 21:19:45 +0100 Subject: [PATCH 4/9] Add missing license header --- x-pack/metricbeat/module/sql/query/query_integration_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 27738142b8a3..eb425ebf47de 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + // +build integration package query From ebe0e7dd4c1ad259c53ff518e6b6909d4b84b593 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 20 Jan 2020 21:26:55 +0100 Subject: [PATCH 5/9] Mask the password in the events --- metricbeat/mb/testing/fetcher.go | 3 +- .../module/sql/query/_meta/data.json | 2 +- .../module/sql/query/_meta/data_postgres.json | 14 ++++---- x-pack/metricbeat/module/sql/query/query.go | 8 +++++ .../sql/query/query_integration_test.go | 34 +++++++++++++++---- 5 files changed, 45 insertions(+), 16 deletions(-) diff --git a/metricbeat/mb/testing/fetcher.go b/metricbeat/mb/testing/fetcher.go index be8264b5cbb5..65ad16807d2d 100644 --- a/metricbeat/mb/testing/fetcher.go +++ b/metricbeat/mb/testing/fetcher.go @@ -26,8 +26,7 @@ import ( // Fetcher is an interface implemented by all fetchers for testing purpouses type Fetcher interface { - Module() mb.Module - Name() string + mb.MetricSet FetchEvents() ([]mb.Event, []error) WriteEvents(testing.TB, string) diff --git a/x-pack/metricbeat/module/sql/query/_meta/data.json b/x-pack/metricbeat/module/sql/query/_meta/data.json index 21531d552dc3..b189b9b11084 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "root:test@tcp(172.22.0.3:3306)/", + "address": "xxxxx", "type": "sql" }, "sql": { diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json index 3bfd814bc732..6c472791b5a9 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "user=postgres password=postgres sslmode=disable host=172.22.0.2 port=5432", + "address": "xxxxx", "type": "sql" }, "sql": { @@ -19,8 +19,8 @@ "numeric": { "blk_read_time": 0, "blk_write_time": 0, - "blks_hit": 4251, - "blks_read": 103, + "blks_hit": 2793, + "blks_read": 116, "conflicts": 0, "datid": 12379, "deadlocks": 0, @@ -28,16 +28,16 @@ "temp_bytes": 0, "temp_files": 0, "tup_deleted": 0, - "tup_fetched": 2847, + "tup_fetched": 1832, "tup_inserted": 0, - "tup_returned": 4877, + "tup_returned": 2898, "tup_updated": 0, - "xact_commit": 49, + "xact_commit": 28, "xact_rollback": 0 }, "string": { "datname": "postgres", - "stats_reset": "2020-01-20 19:48:28.217" + "stats_reset": "2020-01-20 21:02:53.21" } }, "query": "select * from pg_stat_database" diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index b9be69a4d2eb..c56104ec3349 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -63,6 +63,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } +// Host returns the host string that will be stored in the events, as the +// module is generic, the value in `hosts` can contain passwords in different +// places, so mask the whole value. +func (m *MetricSet) Host() string { + // TODO: Return something more meaningful + return "xxxxx" +} + // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index eb425ebf47de..97285e2ea295 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -11,13 +11,16 @@ import ( "net" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" // Drivers _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/tests/compose" + "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/mysql" "github.com/elastic/beats/metricbeat/module/postgresql" @@ -27,14 +30,17 @@ type testFetchConfig struct { Driver string Query string Host string + + Assertion func(t *testing.T, event beat.Event) } func TestMySQL(t *testing.T) { service := compose.EnsureUp(t, "mysql") config := testFetchConfig{ - Driver: "mysql", - Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", - Host: mysql.GetMySQLEnvDSN(service.Host()), + Driver: "mysql", + Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", + Host: mysql.GetMySQLEnvDSN(service.Host()), + Assertion: assertFieldNotContains("service.address", "root:test@"), } t.Run("fetch", func(t *testing.T) { @@ -55,9 +61,10 @@ func TestPostgreSQL(t *testing.T) { password := postgresql.GetEnvPassword() config := testFetchConfig{ - Driver: "postgres", - Query: "select * from pg_stat_database", - Host: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), + Driver: "postgres", + Query: "select * from pg_stat_database", + Host: fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port), + Assertion: assertFieldNotContains("service.address", "password="+password), } t.Run("fetch", func(t *testing.T) { @@ -75,6 +82,12 @@ func testFetch(t *testing.T, cfg testFetchConfig) { require.Empty(t, errs) require.NotEmpty(t, events) t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0]) + + if cfg.Assertion != nil { + for _, event := range events { + cfg.Assertion(t, mbtest.StandardizeEvent(m, event, mb.AddMetricSetInfo)) + } + } } func testData(t *testing.T, cfg testFetchConfig, postfix string) { @@ -91,3 +104,12 @@ func getConfig(cfg testFetchConfig) map[string]interface{} { "sql_query": cfg.Query, } } + +func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event) { + return func(t *testing.T, event beat.Event) { + address, err := event.GetValue(field) + assert.NoError(t, err) + require.NotEmpty(t, address.(string)) + require.NotContains(t, address.(string), s) + } +} From 08eb8e377ebca18070fbbd05e3d415b705a32b6b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 21 Jan 2020 00:00:10 +0100 Subject: [PATCH 6/9] Rename variable in test --- .../metricbeat/module/sql/query/query_integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 97285e2ea295..168d136b0b40 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -107,9 +107,9 @@ func getConfig(cfg testFetchConfig) map[string]interface{} { func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event) { return func(t *testing.T, event beat.Event) { - address, err := event.GetValue(field) + value, err := event.GetValue(field) assert.NoError(t, err) - require.NotEmpty(t, address.(string)) - require.NotContains(t, address.(string), s) + require.NotEmpty(t, value.(string)) + require.NotContains(t, value.(string), s) } } From f3ecb07c6cb7dad35bdff0b36ddbf97f95333030 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 21 Jan 2020 12:26:32 +0100 Subject: [PATCH 7/9] Obtain host from URLs and MySQL DSNs --- vendor/github.com/lib/pq/go.mod | 2 + .../module/sql/query/_meta/data.json | 2 +- .../module/sql/query/_meta/data_postgres.json | 14 +++---- x-pack/metricbeat/module/sql/query/dsn.go | 42 +++++++++++++++++++ x-pack/metricbeat/module/sql/query/query.go | 16 +------ .../sql/query/query_integration_test.go | 13 +++++- 6 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 x-pack/metricbeat/module/sql/query/dsn.go diff --git a/vendor/github.com/lib/pq/go.mod b/vendor/github.com/lib/pq/go.mod index edf0b343fd17..b5a5639ab671 100644 --- a/vendor/github.com/lib/pq/go.mod +++ b/vendor/github.com/lib/pq/go.mod @@ -1 +1,3 @@ module github.com/lib/pq + +go 1.13 diff --git a/x-pack/metricbeat/module/sql/query/_meta/data.json b/x-pack/metricbeat/module/sql/query/_meta/data.json index b189b9b11084..799c66fe7bc9 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "xxxxx", + "address": "172.22.0.3:3306", "type": "sql" }, "sql": { diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json index 6c472791b5a9..0f2db40c5b12 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json @@ -10,7 +10,7 @@ "period": 10000 }, "service": { - "address": "xxxxx", + "address": "172.22.0.2:5432", "type": "sql" }, "sql": { @@ -19,8 +19,8 @@ "numeric": { "blk_read_time": 0, "blk_write_time": 0, - "blks_hit": 2793, - "blks_read": 116, + "blks_hit": 1923, + "blks_read": 111, "conflicts": 0, "datid": 12379, "deadlocks": 0, @@ -28,16 +28,16 @@ "temp_bytes": 0, "temp_files": 0, "tup_deleted": 0, - "tup_fetched": 1832, + "tup_fetched": 1249, "tup_inserted": 0, - "tup_returned": 2898, + "tup_returned": 1356, "tup_updated": 0, - "xact_commit": 28, + "xact_commit": 18, "xact_rollback": 0 }, "string": { "datname": "postgres", - "stats_reset": "2020-01-20 21:02:53.21" + "stats_reset": "2020-01-21 11:23:56.53" } }, "query": "select * from pg_stat_database" diff --git a/x-pack/metricbeat/module/sql/query/dsn.go b/x-pack/metricbeat/module/sql/query/dsn.go new file mode 100644 index 000000000000..2a472c3fbe5b --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/dsn.go @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package query + +import ( + "net/url" + + "github.com/go-sql-driver/mysql" + + "github.com/elastic/beats/metricbeat/mb" +) + +// ParseDSN tries to parse the host +func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { + // TODO: Add support for `username` and `password` as module options + + sanitized := sanitize(host) + + return mb.HostData{ + URI: host, + SanitizedURI: sanitized, + Host: sanitized, + }, nil +} + +func sanitize(host string) string { + // Host is a standard URL + if url, err := url.Parse(host); err == nil && len(url.Host) > 0 { + return url.Host + } + + // Host is a MySQL DSN + if config, err := mysql.ParseDSN(host); err == nil { + return config.Addr + } + + // TODO: Add support for PostgreSQL connection strings and other formats + + return "(redacted)" +} diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index c56104ec3349..3322144d993d 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -10,14 +10,12 @@ import ( "strings" "time" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/mb/parse" - - "github.com/jmoiron/sqlx" ) // init registers the MetricSet with the central registry as soon as the program @@ -26,7 +24,7 @@ import ( // MetricSet has been created then Fetch will begin to be called periodically. func init() { mb.Registry.MustAddMetricSet("sql", "query", New, - mb.WithHostParser(parse.PassThruHostParser), + mb.WithHostParser(ParseDSN), ) } @@ -38,8 +36,6 @@ type MetricSet struct { mb.BaseMetricSet Driver string Query string - - db *sqlx.DB } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -63,14 +59,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { }, nil } -// Host returns the host string that will be stored in the events, as the -// module is generic, the value in `hosts` can contain passwords in different -// places, so mask the whole value. -func (m *MetricSet) Host() string { - // TODO: Return something more meaningful - return "xxxxx" -} - // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 168d136b0b40..9a3fc2609b75 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -40,7 +40,7 @@ func TestMySQL(t *testing.T) { Driver: "mysql", Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", Host: mysql.GetMySQLEnvDSN(service.Host()), - Assertion: assertFieldNotContains("service.address", "root:test@"), + Assertion: assertFieldNotContains("service.address", ":test@"), } t.Run("fetch", func(t *testing.T) { @@ -71,6 +71,17 @@ func TestPostgreSQL(t *testing.T) { testFetch(t, config) }) + config = testFetchConfig{ + Driver: "postgres", + Query: "select * from pg_stat_database", + Host: fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port), + Assertion: assertFieldNotContains("service.address", ":"+password+"@"), + } + + t.Run("fetch with URL", func(t *testing.T) { + testFetch(t, config) + }) + t.Run("data", func(t *testing.T) { testData(t, config, "./_meta/data_postgres.json") }) From 224206e7afd41322f42bddc2bcf1143d4bca4a64 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 21 Jan 2020 12:28:18 +0100 Subject: [PATCH 8/9] Remove unwanted change --- vendor/github.com/lib/pq/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/vendor/github.com/lib/pq/go.mod b/vendor/github.com/lib/pq/go.mod index b5a5639ab671..edf0b343fd17 100644 --- a/vendor/github.com/lib/pq/go.mod +++ b/vendor/github.com/lib/pq/go.mod @@ -1,3 +1 @@ module github.com/lib/pq - -go 1.13 From 7517bfafade43a26a89c29278d0a00f5ae00597b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 21 Jan 2020 12:46:28 +0100 Subject: [PATCH 9/9] Reduce interface changes in Fetcher --- metricbeat/mb/testing/fetcher.go | 17 ++++++++++++++++- .../module/sql/query/query_integration_test.go | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/metricbeat/mb/testing/fetcher.go b/metricbeat/mb/testing/fetcher.go index 65ad16807d2d..5b01e1b81382 100644 --- a/metricbeat/mb/testing/fetcher.go +++ b/metricbeat/mb/testing/fetcher.go @@ -20,17 +20,20 @@ package testing import ( "testing" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" ) // Fetcher is an interface implemented by all fetchers for testing purpouses type Fetcher interface { - mb.MetricSet + Module() mb.Module + Name() string FetchEvents() ([]mb.Event, []error) WriteEvents(testing.TB, string) WriteEventsCond(testing.TB, string, func(common.MapStr) bool) + StandardizeEvent(mb.Event, ...mb.EventModifier) beat.Event } // NewFetcher returns a test fetcher from a Metricset configuration @@ -72,6 +75,10 @@ func (f *reportingMetricSetV2Fetcher) WriteEventsCond(t testing.TB, path string, } } +func (f *reportingMetricSetV2Fetcher) StandardizeEvent(event mb.Event, modifiers ...mb.EventModifier) beat.Event { + return StandardizeEvent(f, event, modifiers...) +} + type reportingMetricSetV2FetcherError struct { mb.ReportingMetricSetV2Error } @@ -95,6 +102,10 @@ func (f *reportingMetricSetV2FetcherError) WriteEventsCond(t testing.TB, path st } } +func (f *reportingMetricSetV2FetcherError) StandardizeEvent(event mb.Event, modifiers ...mb.EventModifier) beat.Event { + return StandardizeEvent(f, event, modifiers...) +} + type reportingMetricSetV2FetcherWithContext struct { mb.ReportingMetricSetV2WithContext } @@ -117,3 +128,7 @@ func (f *reportingMetricSetV2FetcherWithContext) WriteEventsCond(t testing.TB, p t.Fatal("writing events", err) } } + +func (f *reportingMetricSetV2FetcherWithContext) StandardizeEvent(event mb.Event, modifiers ...mb.EventModifier) beat.Event { + return StandardizeEvent(f, event, modifiers...) +} diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 9a3fc2609b75..d7e04f23c406 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -96,7 +96,7 @@ func testFetch(t *testing.T, cfg testFetchConfig) { if cfg.Assertion != nil { for _, event := range events { - cfg.Assertion(t, mbtest.StandardizeEvent(m, event, mb.AddMetricSetInfo)) + cfg.Assertion(t, m.StandardizeEvent(event, mb.AddMetricSetInfo)) } } }