-
Notifications
You must be signed in to change notification settings - Fork 4.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Remove datasource option from SQL module and add tests #15686
Changes from 8 commits
aaadaea
3c8384f
38e8f65
e260032
ebe0e7d
08eb8e3
f3ecb07
224206e
7517bfa
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,3 @@ | ||
This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query. | ||
This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
version: '2.3' | ||
|
||
services: | ||
mysql: | ||
extends: | ||
file: ../../../../metricbeat/module/mysql/docker-compose.yml | ||
service: mysql | ||
|
||
postgresql: | ||
extends: | ||
file: ../../../../metricbeat/module/postgresql/docker-compose.yml | ||
service: postgresql |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,26 +1,30 @@ | ||
{ | ||
"@timestamp":"2016-05-23T08:05:34.853Z", | ||
"beat":{ | ||
"hostname":"beathost", | ||
"name":"beathost" | ||
"@timestamp": "2017-10-12T08:05:34.853Z", | ||
"event": { | ||
"dataset": "sql.query", | ||
"duration": 115000, | ||
"module": "sql" | ||
}, | ||
"metricset":{ | ||
"host":"localhost", | ||
"module":"sql", | ||
"name":"query", | ||
"rtt":44269 | ||
"metricset": { | ||
"name": "query", | ||
"period": 10000 | ||
}, | ||
"sql":{ | ||
"metrics":{ | ||
"numeric":{ | ||
"mynumericfield":1 | ||
}, | ||
"string":{ | ||
"mystringfield":"abc" | ||
} | ||
"service": { | ||
"address": "172.22.0.3:3306", | ||
"type": "sql" | ||
}, | ||
"sql": { | ||
"driver": "mysql", | ||
"metrics": { | ||
"numeric": { | ||
"table_rows": 6 | ||
}, | ||
"string": { | ||
"engine": "InnoDB", | ||
"table_name": "sys_config", | ||
"table_schema": "sys" | ||
} | ||
}, | ||
"driver":"postgres", | ||
"query":"select * from mytable" | ||
}, | ||
"type":"metricsets" | ||
"query": "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows \u003e 0;" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
{ | ||
"@timestamp": "2017-10-12T08:05:34.853Z", | ||
"event": { | ||
"dataset": "sql.query", | ||
"duration": 115000, | ||
"module": "sql" | ||
}, | ||
"metricset": { | ||
"name": "query", | ||
"period": 10000 | ||
}, | ||
"service": { | ||
"address": "172.22.0.2:5432", | ||
"type": "sql" | ||
}, | ||
"sql": { | ||
"driver": "postgres", | ||
"metrics": { | ||
"numeric": { | ||
"blk_read_time": 0, | ||
"blk_write_time": 0, | ||
"blks_hit": 1923, | ||
"blks_read": 111, | ||
"conflicts": 0, | ||
"datid": 12379, | ||
"deadlocks": 0, | ||
"numbackends": 1, | ||
"temp_bytes": 0, | ||
"temp_files": 0, | ||
"tup_deleted": 0, | ||
"tup_fetched": 1249, | ||
"tup_inserted": 0, | ||
"tup_returned": 1356, | ||
"tup_updated": 0, | ||
"xact_commit": 18, | ||
"xact_rollback": 0 | ||
}, | ||
"string": { | ||
"datname": "postgres", | ||
"stats_reset": "2020-01-21 11:23:56.53" | ||
} | ||
}, | ||
"query": "select * from pg_stat_database" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
// or more contributor license agreements. Licensed under the Elastic License; | ||
// you may not use this file except in compliance with the Elastic License. | ||
|
||
package query | ||
|
||
import ( | ||
"net/url" | ||
|
||
"github.com/go-sql-driver/mysql" | ||
|
||
"github.com/elastic/beats/metricbeat/mb" | ||
) | ||
|
||
// ParseDSN tries to parse the host | ||
func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { | ||
// TODO: Add support for `username` and `password` as module options | ||
|
||
sanitized := sanitize(host) | ||
|
||
return mb.HostData{ | ||
URI: host, | ||
SanitizedURI: sanitized, | ||
Host: sanitized, | ||
}, nil | ||
} | ||
|
||
func sanitize(host string) string { | ||
// Host is a standard URL | ||
if url, err := url.Parse(host); err == nil && len(url.Host) > 0 { | ||
return url.Host | ||
} | ||
|
||
// Host is a MySQL DSN | ||
if config, err := mysql.ParseDSN(host); err == nil { | ||
return config.Addr | ||
} | ||
|
||
// TODO: Add support for PostgreSQL connection strings and other formats | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. would this mean that we need to add support for most existing db engines? This wasn't the case with the previous There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is only needed to have a proper value in With the I.e. with this configuration:
The module was connecting to With the changes in this PR, the configuration has to be like this:
In this case we need to parse this DSN to get the real address and don't expose passwords. Different drivers accept different DSN formats. This is why we may need multiple parsers here. This doesn't affect data collection, data collection will work with any driver and DSN format even if we cannot parse it here. |
||
|
||
return "(redacted)" | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
TODOs added to #15048, not needed for initial beta release.