From e3e6bd786dab165bb4e46248161c656c54486753 Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Thu, 7 Nov 2024 05:41:48 +1100
Subject: [PATCH] [8.16] [Observability] Fix integrations detected twice
 (#198670) (#199183)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[Observability] Fix integrations detected twice
(#198670)](https://github.com/elastic/kibana/pull/198670)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Thom
Heymann","email":"190132+thomheymann@users.noreply.github.com"},"sourceCommit":{"committedDate":"2024-11-06T16:55:33Z","message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<johannes.reuter@elastic.co>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v9.0.0","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-logs","Feature:
Observability Onboarding","v8.16.0","v8.17.0"],"title":"[Observability]
Fix integrations detected
twice","number":198670,"url":"https://github.com/elastic/kibana/pull/198670","mergeCommit":{"message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<johannes.reuter@elastic.co>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121"}},"sourceBranch":"main","suggestedTargetBranches":["8.16","8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/198670","number":198670,"mergeCommit":{"message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<johannes.reuter@elastic.co>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Thom Heymann <190132+thomheymann@users.noreply.github.com>
---
 .../public/assets/auto_detect.sh              | 30 ++++++++++++++-----
 .../public/assets/integrations.conf           |  2 +-
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
index ebdcdeb0d81dc..c13b5cf031e0d 100755
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh
@@ -311,15 +311,31 @@ apply_elastic_agent_config() {
 
 read_open_log_file_list() {
   local exclude_patterns=(
-    "^\/Users\/.+?\/Library\/Application Support"
-    "^\/Users\/.+?\/Library\/Group Containers"
-    "^\/Users\/.+?\/Library\/Containers"
-    "^\/Users\/.+?\/Library\/Caches"
-    "^\/private"
+    "^\/Users\/.+?\/Library\/Application Support\/"
+    "^\/Users\/.+?\/Library\/Group Containers\/"
+    "^\/Users\/.+?\/Library\/Containers\/"
+    "^\/Users\/.+?\/Library\/Caches\/"
+    "^\/private\/"
+
+    # Integrations only ingest a subset of application logs so there are scenarios where additional
+    # log files could be detected and displayed as a "custom log" alongside the detected integration
+    # they belong to. To avoid this UX issue we exclude all log files inside application directories
+    # from the custom log file detection
+    "^\/var\/log\/nginx\/"
+    "^\/var\/log\/apache2\/"
+    "^\/var\/log\/httpd\/"
+    "^\/var\/log\/mysql\/"
+    "^\/var\/log\/postgresql\/"
+    "^\/var\/log\/redis\/"
+    "^\/var\/log\/rabbitmq\/"
+    "^\/var\/log\/kafka\/"
+    "^\/var\/log\/mongodb\/"
+    "^\/opt\/tomcat\/logs\/"
+    "^\/var\/log\/prometheus\/"
 
     # Exclude previous installation logs
-    "\/opt\/Elastic\/Agent\/"
-    "\/Library\/Elastic\/Agent\/"
+    "^\/opt\/Elastic\/Agent\/"
+    "^\/Library\/Elastic\/Agent\/"
   )
 
   # Excluding all patterns that correspond to known integrations
diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
index e6455a9170c86..e9afdc3cdfb2e 100644
--- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
+++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf
@@ -34,9 +34,9 @@ patterns=
 title=PostgreSQL Logs
 patterns=
     /var/log/postgresql/postgresql-*-*.log*
+    /var/log/postgresql/postgresql-*-*.csv*
     /*/postgresql-logs/*.log
     /etc/postgresql/*/main/postgresql.conf
-    /var/log/postgresql/postgresql-*-*.csv*
 
 [redis]
 title=Redis Logs