From 7be50c0586389e7012b857083b2880861d53d4b6 Mon Sep 17 00:00:00 2001 From: Agi K Thomas <101976829+agithomas@users.noreply.github.com> Date: Thu, 17 Nov 2022 11:31:07 +0530 Subject: [PATCH] Feature/32089 generic sql enchancement (#32293) * Upgrade Generic SQL to support new connection string and special character support --- CHANGELOG.next.asciidoc | 2 +- metricbeat/docs/modules/sql.asciidoc | 83 ++++++++++++++++++- .../metricbeat/module/sql/_meta/docs.asciidoc | 83 ++++++++++++++++++- .../metricbeat/module/sql/docker-compose.yml | 5 ++ x-pack/metricbeat/module/sql/query/dsn.go | 37 ++++++++- .../sql/query/query_integration_test.go | 77 ++++++++++++++++- .../module/sql/query/test_sql_oracle.py | 50 +++++++++++ 7 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 x-pack/metricbeat/module/sql/query/test_sql_oracle.py diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6b4dac64125..68b19b30c9e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -95,7 +95,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263] - Add tags to events based on parsed identifier. {pull}33472[33472] - Skip over unsupported filesystems in the system.filesystem metricset instead of failing immediately. Fix debug statement in system.fsstat metricset. {pull}33646[33646] - +- Support Oracle-specific connection strings in SQL module {issue}32089[32089] {pull}32293[32293] *Packetbeat* diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 987bc5556e7..229c00960ca 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -52,6 +52,11 @@ each row. `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding `sql_response_format`: either `variables` or `table` +[float] +== Example + +Examples of configurations in `sql.yml` to connect with supported databases are mentioned below. + [float] === Example: capture Innodb-related metrics @@ -238,7 +243,7 @@ This `sql.yml` configuration shows how to get the buffer cache hit ratio: metricsets: - query period: 10s - hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["oracle://sys:password@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] driver: "oracle" sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' @@ -298,6 +303,8 @@ The example generates this event: "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" } } + + ---- [float] @@ -551,6 +558,80 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi } ---- +=== Host Setup + +Some drivers require additional configuration to work. Find here instructions for these drivers. + +==== Oracle Database Connection Pre-requisites + +To get connected with the Oracle Database `ORACLE_SID`, `ORACLE_BASE`, `ORACLE_HOME` environment variables should be set. + +For example: Let us consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows: + +[source,bash] +---- +export ORACLE_BASE=/opt/oracle/oradata +export ORACLE_HOME=/opt/oracle/product/21c/dbhome_1 +---- +Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. + +===== Oracle Instant Client Installation + +Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation. + +The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here]. + +===== Enable Oracle Listener + +The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. +To check if the listener is running or not, run: + +[source,bash] +---- +lsnrctl STATUS +---- + +If the listener is not running, use the command to start: + +[source,bash] +---- +lsnrctl START +---- + +Then, Metricbeat can be launched. + +===== Host Configuration for Oracle + +The following two types of host configurations are supported: + +1. DSN host configuration as URL: + a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` + b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` + +2. DSN host configuration: + a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` + b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` + +Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. + +Username and Password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. + +[source,yml] +---- +- module: sql + metricsets: + - query + period: 10s + driver: "oracle" + enabled: true + hosts: ['user="" password="" connectString="0.0.0.0:1521/ORCLCDB.localdomain" sysdba=true'] + username: sys + password: password + sql_queries: + - query: SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 and METRIC_NAME LIKE '%' + response_format: variables +---- + :edit_url: [float] diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index 08deea8196e..9286e043d60 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -38,6 +38,11 @@ each row. `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding `sql_response_format`: either `variables` or `table` +[float] +== Example + +Examples of configurations in `sql.yml` to connect with supported databases are mentioned below. + [float] === Example: capture Innodb-related metrics @@ -224,7 +229,7 @@ This `sql.yml` configuration shows how to get the buffer cache hit ratio: metricsets: - query period: 10s - hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + hosts: ["oracle://sys:password@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] driver: "oracle" sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' @@ -284,6 +289,8 @@ The example generates this event: "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" } } + + ---- [float] @@ -535,4 +542,78 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi "dataset": "sql.query" } } +---- + +=== Host Setup + +Some drivers require additional configuration to work. Find here instructions for these drivers. + +==== Oracle Database Connection Pre-requisites + +To get connected with the Oracle Database `ORACLE_SID`, `ORACLE_BASE`, `ORACLE_HOME` environment variables should be set. + +For example: Let us consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows: + +[source,bash] +---- +export ORACLE_BASE=/opt/oracle/oradata +export ORACLE_HOME=/opt/oracle/product/21c/dbhome_1 +---- +Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. + +===== Oracle Instant Client Installation + +Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation. + +The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here]. + +===== Enable Oracle Listener + +The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. +To check if the listener is running or not, run: + +[source,bash] +---- +lsnrctl STATUS +---- + +If the listener is not running, use the command to start: + +[source,bash] +---- +lsnrctl START +---- + +Then, Metricbeat can be launched. + +===== Host Configuration for Oracle + +The following two types of host configurations are supported: + +1. DSN host configuration as URL: + a. `hosts: ["user/pass@0.0.0.0:1521/ORCLPDB1.localdomain"]` + b. `hosts: ["user/password@0.0.0.0:1521/ORCLPDB1.localdomain as sysdba"]` + +2. DSN host configuration: + a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']` + b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']` + +Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`. + +Username and Password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. + +[source,yml] +---- +- module: sql + metricsets: + - query + period: 10s + driver: "oracle" + enabled: true + hosts: ['user="" password="" connectString="0.0.0.0:1521/ORCLCDB.localdomain" sysdba=true'] + username: sys + password: password + sql_queries: + - query: SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 and METRIC_NAME LIKE '%' + response_format: variables ---- \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/docker-compose.yml b/x-pack/metricbeat/module/sql/docker-compose.yml index a053c322d30..fac50c274b1 100644 --- a/x-pack/metricbeat/module/sql/docker-compose.yml +++ b/x-pack/metricbeat/module/sql/docker-compose.yml @@ -10,3 +10,8 @@ services: extends: file: ../../../../metricbeat/module/postgresql/docker-compose.yml service: postgresql + + oracle: + extends: + file: ../../../../x-pack/metricbeat/module/oracle/docker-compose.yml + service: oracle diff --git a/x-pack/metricbeat/module/sql/query/dsn.go b/x-pack/metricbeat/module/sql/query/dsn.go index 0ff3796da1c..cad3f3181b3 100644 --- a/x-pack/metricbeat/module/sql/query/dsn.go +++ b/x-pack/metricbeat/module/sql/query/dsn.go @@ -5,19 +5,52 @@ package query import ( + "fmt" "net/url" "github.com/go-sql-driver/mysql" + "github.com/godror/godror" + "github.com/godror/godror/dsn" "github.com/elastic/beats/v7/metricbeat/mb" ) +// ConnectionDetails contains all possible data that can be used to create a connection with +// an Oracle db +type ConnectionDetails struct { + Username string `config:"username"` + Password string `config:"password"` + Driver string `config:"driver"` +} + // ParseDSN tries to parse the host func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { // TODO: Add support for `username` and `password` as module options - + config := ConnectionDetails{} + if err := mod.UnpackConfig(&config); err != nil { + return mb.HostData{}, fmt.Errorf("error parsing config file: %w", err) + } + if config.Driver == "oracle" { + params, err := godror.ParseDSN(host) + if err != nil { + return mb.HostData{}, fmt.Errorf("error trying to parse connection string in field 'hosts': %w", err) + } + if params.Username == "" { + params.Username = config.Username + } + if params.Password.Secret() == "" { + params.StandaloneConnection = true + params.Password = dsn.NewPassword(config.Password) + } + return mb.HostData{ + URI: params.StringWithPassword(), + SanitizedURI: params.ConnectString, + Host: params.String(), + User: params.Username, + Password: params.Password.Secret(), + }, nil + } sanitized := sanitize(host) - return mb.HostData{ URI: host, SanitizedURI: sanitized, diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index fb7692c17c0..81e87b0491d 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -10,8 +10,11 @@ package query import ( "fmt" "net" + "os" "testing" + "time" + "github.com/godror/godror" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -40,7 +43,6 @@ func TestMySQL(t *testing.T) { Driver: "mysql", Query: "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;", ResponseFormat: tableResponseFormat, - RawData: rawData{ Enabled: true, }, @@ -48,7 +50,6 @@ func TestMySQL(t *testing.T) { Host: mysql.GetMySQLEnvDSN(service.Host()), Assertion: assertFieldNotContains("service.address", ":test@"), } - t.Run("fetch", func(t *testing.T) { testFetch(t, cfg) }) @@ -226,6 +227,26 @@ func TestPostgreSQL(t *testing.T) { }) } +func TestOracle(t *testing.T) { + service := compose.EnsureUp(t, "oracle") + host, port, _ := net.SplitHostPort(service.Host()) + cfg := testFetchConfig{ + config: config{ + Driver: "oracle", + Query: `SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit_Ratio" FROM V$BUFFER_POOL_STATISTICS`, + ResponseFormat: tableResponseFormat, + RawData: rawData{ + Enabled: true, + }, + }, + Host: GetOracleConnectionDetails(t, host, port), + Assertion: assertFieldContainsFloat64("Hit_Ratio", 0.0), + } + t.Run("fetch", func(t *testing.T) { + testFetch(t, cfg) + }) +} + func testFetch(t *testing.T, cfg testFetchConfig) { m := mbtest.NewFetcher(t, getConfig(cfg)) events, errs := m.FetchEvents() @@ -256,11 +277,9 @@ func getConfig(cfg testFetchConfig) map[string]interface{} { "raw_data.enabled": cfg.config.RawData.Enabled, "merge_results": cfg.config.MergeResults, } - if cfg.config.ResponseFormat != "" { values["sql_response_format"] = cfg.config.ResponseFormat } - return values } @@ -272,3 +291,53 @@ func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event require.NotContains(t, value.(string), s) } } + +func assertFieldContainsFloat64(field string, limit float64) func(t *testing.T, event beat.Event) { + return func(t *testing.T, event beat.Event) { + value, err := event.GetValue("sql.metrics.hit_ratio") + assert.NoError(t, err) + require.GreaterOrEqual(t, value.(float64), limit) + } +} + +func GetOracleConnectionDetails(t *testing.T, host string, port string) string { + params, err := godror.ParseDSN(GetOracleConnectString(host, port)) + require.Empty(t, err) + return params.StringWithPassword() +} + +// GetOracleEnvServiceName returns the service name to use with Oracle testing server or the value of the environment variable ORACLE_SERVICE_NAME if not empty +func GetOracleEnvServiceName() string { + serviceName := os.Getenv("ORACLE_SERVICE_NAME") + if len(serviceName) == 0 { + serviceName = "ORCLCDB.localdomain" + } + return serviceName +} + +// GetOracleEnvUsername returns the username to use with Oracle testing server or the value of the environment variable ORACLE_USERNAME if not empty +func GetOracleEnvUsername() string { + username := os.Getenv("ORACLE_USERNAME") + if len(username) == 0 { + username = "sys" + } + return username +} + +// GetOracleEnvUsername returns the port of the Oracle server or the value of the environment variable ORACLE_PASSWORD if not empty +func GetOracleEnvPassword() string { + password := os.Getenv("ORACLE_PASSWORD") + if len(password) == 0 { + password = "Oradoc_db1" // #nosec + } + return password +} + +func GetOracleConnectString(host string, port string) string { + time.Sleep(300 * time.Second) + connectString := os.Getenv("ORACLE_CONNECT_STRING") + if len(connectString) == 0 { + connectString = fmt.Sprintf("%s/%s@%s:%s/%s as sysdba", GetOracleEnvUsername(), GetOracleEnvPassword(), host, port, GetOracleEnvServiceName()) + } + return connectString +} diff --git a/x-pack/metricbeat/module/sql/query/test_sql_oracle.py b/x-pack/metricbeat/module/sql/query/test_sql_oracle.py new file mode 100644 index 00000000000..fd1cfc85fd1 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/test_sql_oracle.py @@ -0,0 +1,50 @@ +import os +import sys +import unittest +import time +from xpack_metricbeat import XPackTest, metricbeat + + +class Test(XPackTest): + + COMPOSE_SERVICES = ['oracle'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_query(self): + """ + sql oracle custom query test + """ + self.render_config_template(modules=[{ + "name": "sql", + "metricsets": ["query"], + "hosts": self.get_hosts(), + "period": "5s", + "additional_content": """ + driver: oracle + sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) FROM V$BUFFER_POOL_STATISTICS' + sql_response_format: table""" + }]) + proc = self.start_beat(home=self.beat_path) + self.wait_until(lambda: self.output_lines() > 0) + self.wait_until(lambda: self.check_for_events(), max_timeout=300) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertGreater(len(output), 0) + + event_valid_counter = False + for evt in output: + if evt.get("sql") and evt["sql"].get("query"): + event_valid_counter = True + + def check_for_events(self): + output = self.read_output_json() + for evt in output: + if evt.get("sql") and evt["sql"].get("query"): + return True + + return False + + def get_hosts(self): + return ['user="sys" password="Oradoc_db1" connectString="{}/ORCLPDB1.localdomain" sysdba=true'.format(self.compose_host())]