[8.16] [Observability] Fix integrations detected twice (#198670) (#19…

…9183)

# Backport

This will backport the following commits from `main` to `8.16`:
- [[Observability] Fix integrations detected twice
(#198670)](#198670)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Thom
Heymann","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-06T16:55:33Z","message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<[email protected]>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","v9.0.0","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-logs","Feature:
Observability Onboarding","v8.16.0","v8.17.0"],"title":"[Observability]
Fix integrations detected
twice","number":198670,"url":"https://github.com/elastic/kibana/pull/198670","mergeCommit":{"message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<[email protected]>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121"}},"sourceBranch":"main","suggestedTargetBranches":["8.16","8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/198670","number":198670,"mergeCommit":{"message":"[Observability]
Fix integrations detected twice (#198670)\n\nResolves
[#195912](https://github.com/elastic/kibana/issues/195912)\r\n\r\n##
Summary\r\n\r\nFixes an issue where integrations were displayed twice in
the list of\r\ndetected log files.\r\n\r\nCo-authored-by: Joe Reuter
<[email protected]>","sha":"f410085ffc6e37f284ff72b034ca177f536f7121"}},{"branch":"8.16","label":"v8.16.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Thom Heymann <[email protected]>

x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh

@@ -311,15 +311,31 @@ apply_elastic_agent_config() {
 
 read_open_log_file_list() {
   local exclude_patterns=(
-    "^\/Users\/.+?\/Library\/Application Support"
-    "^\/Users\/.+?\/Library\/Group Containers"
-    "^\/Users\/.+?\/Library\/Containers"
-    "^\/Users\/.+?\/Library\/Caches"
-    "^\/private"
+    "^\/Users\/.+?\/Library\/Application Support\/"
+    "^\/Users\/.+?\/Library\/Group Containers\/"
+    "^\/Users\/.+?\/Library\/Containers\/"
+    "^\/Users\/.+?\/Library\/Caches\/"
+    "^\/private\/"
+
+    # Integrations only ingest a subset of application logs so there are scenarios where additional
+    # log files could be detected and displayed as a "custom log" alongside the detected integration
+    # they belong to. To avoid this UX issue we exclude all log files inside application directories
+    # from the custom log file detection
+    "^\/var\/log\/nginx\/"
+    "^\/var\/log\/apache2\/"
+    "^\/var\/log\/httpd\/"
+    "^\/var\/log\/mysql\/"
+    "^\/var\/log\/postgresql\/"
+    "^\/var\/log\/redis\/"
+    "^\/var\/log\/rabbitmq\/"
+    "^\/var\/log\/kafka\/"
+    "^\/var\/log\/mongodb\/"
+    "^\/opt\/tomcat\/logs\/"
+    "^\/var\/log\/prometheus\/"
 
     # Exclude previous installation logs
-    "\/opt\/Elastic\/Agent\/"
-    "\/Library\/Elastic\/Agent\/"
+    "^\/opt\/Elastic\/Agent\/"
+    "^\/Library\/Elastic\/Agent\/"
   )
 
   # Excluding all patterns that correspond to known integrations

x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf

@@ -34,9 +34,9 @@ patterns=
 title=PostgreSQL Logs
 patterns=
     /var/log/postgresql/postgresql-*-*.log*
+    /var/log/postgresql/postgresql-*-*.csv*
     /*/postgresql-logs/*.log
     /etc/postgresql/*/main/postgresql.conf
-    /var/log/postgresql/postgresql-*-*.csv*
 
 [redis]
 title=Redis Logs