Feature/32089 generic sql enchancement (#32293)

* Upgrade Generic SQL to support new connection string and special character support

CHANGELOG.next.asciidoc

@@ -95,7 +95,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263]
 - Add tags to events based on parsed identifier. {pull}33472[33472]
 - Skip over unsupported filesystems in the system.filesystem metricset instead of failing immediately. Fix debug statement in system.fsstat metricset. {pull}33646[33646]
-
+- Support Oracle-specific connection strings in SQL module {issue}32089[32089] {pull}32293[32293]
 
 *Packetbeat*
 

metricbeat/docs/modules/sql.asciidoc

@@ -52,6 +52,11 @@ each row.
 
 `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding  `sql_response_format`: either `variables` or `table`
 
+[float]
+== Example
+
+Examples of configurations in `sql.yml` to connect with supported databases are mentioned below. 
+
 [float]
 === Example: capture Innodb-related metrics
 
@@ -238,7 +243,7 @@ This `sql.yml` configuration shows how to get the buffer cache hit ratio:
   metricsets:
     - query
   period: 10s
-  hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"]
+  hosts: ["oracle://sys:password@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"]
 
   driver: "oracle"
   sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS'
@@ -298,6 +303,8 @@ The example generates this event:
     "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788"
   }
 }
+
+
 ----
 
 [float]
@@ -551,6 +558,80 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi
 }
 ----
 
+=== Host Setup
+
+Some drivers require additional configuration to work. Find here instructions for these drivers.
+
+==== Oracle Database Connection Pre-requisites
+
+To get connected with the Oracle Database `ORACLE_SID`, `ORACLE_BASE`, `ORACLE_HOME` environment variables should be set.
+
+For example: Let us consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows:
+
+[source,bash]
+----
+export ORACLE_BASE=/opt/oracle/oradata
+export ORACLE_HOME=/opt/oracle/product/21c/dbhome_1
+----
+Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. 
+
+===== Oracle Instant Client Installation
+
+Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation.
+
+The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here].
+
+===== Enable Oracle Listener
+
+The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. 
+To check if the listener is running or not, run: 
+
+[source,bash]
+----
+lsnrctl STATUS
+----
+
+If the listener is not running, use the command to start:
+
+[source,bash]
+----
+lsnrctl START
+----
+
+Then, Metricbeat can be launched.
+
+===== Host Configuration for Oracle
+
+The following two types of host configurations are supported:
+
+1. DSN host configuration as URL:
+    a. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain"]`
+    b. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain as sysdba"]`
+
+2. DSN host configuration:
+    a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']`
+    b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']`
+
+Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`.
+
+Username and Password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. 
+
+[source,yml]
+----
+- module: sql
+  metricsets:
+    - query
+  period: 10s
+  driver: "oracle"
+  enabled: true
+  hosts: ['user="" password="" connectString="0.0.0.0:1521/ORCLCDB.localdomain" sysdba=true']
+  username: sys
+  password: password
+  sql_queries: 
+  - query: SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 and METRIC_NAME LIKE '%'
+    response_format: variables 
+----
+
 :edit_url:
 
 [float]

x-pack/metricbeat/module/sql/_meta/docs.asciidoc

@@ -38,6 +38,11 @@ each row.
 
 `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding  `sql_response_format`: either `variables` or `table`
 
+[float]
+== Example
+
+Examples of configurations in `sql.yml` to connect with supported databases are mentioned below. 
+
 [float]
 === Example: capture Innodb-related metrics
 
@@ -224,7 +229,7 @@ This `sql.yml` configuration shows how to get the buffer cache hit ratio:
   metricsets:
     - query
   period: 10s
-  hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"]
+  hosts: ["oracle://sys:password@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"]
 
   driver: "oracle"
   sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS'
@@ -284,6 +289,8 @@ The example generates this event:
     "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788"
   }
 }
+
+
 ----
 
 [float]
@@ -535,4 +542,78 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi
     "dataset": "sql.query"
   }
 }
+----
+
+=== Host Setup
+
+Some drivers require additional configuration to work. Find here instructions for these drivers.
+
+==== Oracle Database Connection Pre-requisites
+
+To get connected with the Oracle Database `ORACLE_SID`, `ORACLE_BASE`, `ORACLE_HOME` environment variables should be set.
+
+For example: Let us consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows:
+
+[source,bash]
+----
+export ORACLE_BASE=/opt/oracle/oradata
+export ORACLE_HOME=/opt/oracle/product/21c/dbhome_1
+----
+Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. 
+
+===== Oracle Instant Client Installation
+
+Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation.
+
+The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here].
+
+===== Enable Oracle Listener
+
+The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. 
+To check if the listener is running or not, run: 
+
+[source,bash]
+----
+lsnrctl STATUS
+----
+
+If the listener is not running, use the command to start:
+
+[source,bash]
+----
+lsnrctl START
+----
+
+Then, Metricbeat can be launched.
+
+===== Host Configuration for Oracle
+
+The following two types of host configurations are supported:
+
+1. DSN host configuration as URL:
+    a. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain"]`
+    b. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain as sysdba"]`
+
+2. DSN host configuration:
+    a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']`
+    b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']`
+
+Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`.
+
+Username and Password to connect to the database can be provided as values to `username` and `password` keys of `sql.yml`. 
+
+[source,yml]
+----
+- module: sql
+  metricsets:
+    - query
+  period: 10s
+  driver: "oracle"
+  enabled: true
+  hosts: ['user="" password="" connectString="0.0.0.0:1521/ORCLCDB.localdomain" sysdba=true']
+  username: sys
+  password: password
+  sql_queries: 
+  - query: SELECT METRIC_NAME, VALUE FROM V$SYSMETRIC WHERE GROUP_ID = 2 and METRIC_NAME LIKE '%'
+    response_format: variables 
 ----

x-pack/metricbeat/module/sql/docker-compose.yml

@@ -10,3 +10,8 @@ services:
     extends:
       file: ../../../../metricbeat/module/postgresql/docker-compose.yml
       service: postgresql
+
+  oracle:
+    extends:
+      file: ../../../../x-pack/metricbeat/module/oracle/docker-compose.yml
+      service: oracle

x-pack/metricbeat/module/sql/query/dsn.go

@@ -5,19 +5,52 @@
 package query
 
 import (
+	"fmt"
 	"net/url"
 
 	"github.com/go-sql-driver/mysql"
+	"github.com/godror/godror"
+	"github.com/godror/godror/dsn"
 
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
+// ConnectionDetails contains all possible data that can be used to create a connection with
+// an Oracle db
+type ConnectionDetails struct {
+	Username string `config:"username"`
+	Password string `config:"password"`
+	Driver   string `config:"driver"`
+}
+
 // ParseDSN tries to parse the host
 func ParseDSN(mod mb.Module, host string) (mb.HostData, error) {
 	// TODO: Add support for `username` and `password` as module options
-
+	config := ConnectionDetails{}
+	if err := mod.UnpackConfig(&config); err != nil {
+		return mb.HostData{}, fmt.Errorf("error parsing config file: %w", err)
+	}
+	if config.Driver == "oracle" {
+		params, err := godror.ParseDSN(host)
+		if err != nil {
+			return mb.HostData{}, fmt.Errorf("error trying to parse connection string in field 'hosts': %w", err)
+		}
+		if params.Username == "" {
+			params.Username = config.Username
+		}
+		if params.Password.Secret() == "" {
+			params.StandaloneConnection = true
+			params.Password = dsn.NewPassword(config.Password)
+		}
+		return mb.HostData{
+			URI:          params.StringWithPassword(),
+			SanitizedURI: params.ConnectString,
+			Host:         params.String(),
+			User:         params.Username,
+			Password:     params.Password.Secret(),
+		}, nil
+	}
 	sanitized := sanitize(host)
-
 	return mb.HostData{
 		URI:          host,
 		SanitizedURI: sanitized,