Backport changes from 2012 and 2017 (opensearch-project#2020)

Signed-off-by: Adam Tackett <[email protected]> Co-authored-by: Adam Tackett <[email protected]>
mengweieric · Jul 31, 2024 · a939635 · a939635
1 parent 6094e4c
commit a939635
Show file tree

Hide file tree

Showing 15 changed files with 104 additions and 91 deletions.
diff --git a/public/components/getting_started/getting_started_artifacts/csv_file/csv_file-1.0.0.json b/public/components/getting_started/getting_started_artifacts/csv_file/csv_file-1.0.0.json
@@ -7,7 +7,7 @@
   "type": "logs",
   "labels": ["Logs", "Unstructured"],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/csv_file/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file",
   "workflows": [
     {
       "name": "dashboards",
@@ -40,7 +40,7 @@
   },
   "components": [],
   "getting-started": {
-    "info": "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/info/Getting-Started.md",
+    "info": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/getting-started/Getting-Started.md",
     "ingestion": ["fluent-bit","data-prepper"],
     "structured": "true",
     "technology": "csv",
@@ -76,29 +76,29 @@
                 "type": "host-name"
               }
             ],
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/csv_file/getting-started/fluent-bit/.env"],
-            "content": "wget https://raw.githubusercontent.com/opensearch-catalog/tree/main/integrations/csv_file/getting-started/fluent-bit/.env",
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit/.env"],
+            "content": "wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/fluent-bit/.env",
             "description": "Setup docker-compose env variables "
           },
           {
             "name": "Setup fluent-bit folder",
             "label": "Environment Parameters",
             "phase": "docker",
             "type": "file-url",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/csv_file/getting-started/fluent-bit/data/fluent-bit.conf",
-              "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/csv_file/getting-started/fluent-bit/data/parsers.conf"
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf",
+              "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf"
             ],
-            "content": "wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/csv_file/getting-started/fluent-bit/data/fluent-bit.conf \\\n     https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf\n",
+            "content": "mkdir -p fluent-bit\nwget -P fluent-bit  https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf \\\n     https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf\n",
             "description": "Setup docker-compose fluent-bit's service configuration "
           },
           {
             "name": "Run docker-compose",
             "label": "live container",
             "type": "file-url",
             "phase": "docker",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit.yml"],
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.yml"],
             "description": "Run docker-compose for the nginx live example and see the generated index",
-            "content": "wget -O fluent-bit.yml https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/fluent-bit.yml && docker-compose -f fluent-bit.yml up -d\n"
+            "content": "wget -O docker-compose.yml https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/csv_file/getting-started/docker-compose.yml && docker-compose up -d fluent-bit \n"
           }
         ]
       },
@@ -111,7 +111,7 @@
             "type": "console-cmd",
             "phase": "ingestion",
             "label": "Log Parsing",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit/data/parsers.conf"],
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/getting-started/fluent-bit/parsers.conf"],
             "description": "Setup Fluent-Bit parser config file parsing csv file",
             "content": "[PARSER]\n    Name        csv\n    Format      regex\n    Regex       ^(?<timestamp>[^,]+),(?<log_level>[^,]+),(?<message>[^,]+),(?<application>[^,]+),(?<host>[^,]+)$\n    Time_Key    timestamp\n    Time_Format %Y-%m-%d %H:%M:%S\n"
           },
@@ -120,7 +120,7 @@
             "type": "console-cmd",
             "phase": "ingestion",
             "label": "Agent Set-Up",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/getting-started/fluent-bit/data/fluent-bit.conf"],
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/getting-started/fluent-bit/fluent-bit.conf"],
             "description": "Setup Fluent-Bit conf file including logs parsing and OpenSearch access",
             "input-params": [
               {
@@ -145,15 +145,15 @@
         "info": ["https://opensearch.org/docs/latest/im-plugin/index-templates"],
         "content": "PUT _index_template/csv_logs_template",
         "description": "Setup sample csv based schema logs index template in the dev console",
-        "index-template": "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/schema/csv-logs-1.0.0.mapping.json",
+        "index-template": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/schema/csv-logs-1.0.0.mapping.json",
         "index-pattern-name": "logs-*"
       }
     ],
     "index-patterns": {
       "type": ["logs"],
       "info": ["https://opensearch.org/docs/latest/dashboards/management/index-patterns/"],
       "description": "Import index patterns `.ndjson` file to the saved objects",
-      "index-pattern": "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/csv_file/assets/fluent-bit-csv-upload-1.0.0.ndjson",
+      "index-pattern": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/csv_file/assets/fluent-bit-csv-upload-1.0.0.ndjson",
       "index-patterns-name": ["logs-*"]
     }
   },

diff --git a/.../csv_file/getting-started/fluent-bit/.env → ...d_artifacts/csv_file/getting-started/.env b/.../csv_file/getting-started/fluent-bit/.env → ...d_artifacts/csv_file/getting-started/.env
diff --git a/...ng-started/fluent-bit/docker-complete.yml → ..._file/getting-started/docker-complete.yml b/...ng-started/fluent-bit/docker-complete.yml → ..._file/getting-started/docker-complete.yml
@@ -40,13 +40,13 @@ services:
       - opensearch-net
 
   fluent-bit:
-    image: fluent/fluent-bit:latest
     container_name: fluent-bit
     volumes:
-      - ./data:/fluent-bit/data
-    command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"]
-    depends_on:
-      - opensearch-node1
+      - ./logs:/logs
+      - ./fluent-bit:/fluent-bit/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
     networks:
       - opensearch-net
 

diff --git a/...getting-started/fluent-bit/fluent-bit.yml → ...s/csv_file/getting-started/fluent-bit.yml b/...getting-started/fluent-bit/fluent-bit.yml → ...s/csv_file/getting-started/fluent-bit.yml
@@ -2,11 +2,14 @@ version: '3.8'
 
 services:
   fluent-bit:
-    image: fluent/fluent-bit:latest
     container_name: fluent-bit
+    image: fluent/fluent-bit:latest
     volumes:
-      - ./data:/fluent-bit/data
-    command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/data/fluent-bit.conf"]
+      - ./logs:/logs
+      - ./fluent-bit:/fluent-bit/etc
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
     networks:
       - opensearch-net
 

diff --git a/...g-started/fluent-bit/data/fluent-bit.conf → ...etting-started/fluent-bit/fluent-bit.conf b/...g-started/fluent-bit/data/fluent-bit.conf → ...etting-started/fluent-bit/fluent-bit.conf
@@ -5,7 +5,7 @@
 
 [INPUT]
     Name         tail
-    Path         /fluent-bit/data/*.csv
+    Path         *.csv
     Parser       csv
     Tag          csv
 
@@ -25,4 +25,4 @@
     tls.verify      Off
     Suppress_Type_Name On
     HTTP_User       admin
-    HTTP_Passwd     my_%New%_passW0rd!@#
+    HTTP_Passwd     my_%New%_passW0rd!@#
diff --git a/.../getting-started/fluent-bit/data/logs.csv → ..._file/getting-started/fluent-bit/logs.csv b/.../getting-started/fluent-bit/data/logs.csv → ..._file/getting-started/fluent-bit/logs.csv
diff --git a/...ting-started/fluent-bit/data/parsers.conf → ...e/getting-started/fluent-bit/parsers.conf b/...ting-started/fluent-bit/data/parsers.conf → ...e/getting-started/fluent-bit/parsers.conf
diff --git a/...mponents/getting_started/getting_started_artifacts/csv_file/getting-started/logs/logs.csv b/...mponents/getting_started/getting_started_artifacts/csv_file/getting-started/logs/logs.csv
@@ -0,0 +1,10 @@
+2024-07-16 12:00:00,INFO,Application started successfully,App1,host1
+2024-07-16 12:01:00,DEBUG,User logged in,App1,host1
+2024-07-16 12:01:05,ERROR,Failed to load resource,App1,host1
+2024-07-16 12:02:00,WARN,Deprecated API used,App1,host1
+2024-07-16 12:03:00,INFO,Background job executed,App1,host1
+2024-07-16 12:04:00,DEBUG,Cache cleared,App1,host1
+2024-07-16 12:05:00,INFO,User logged out,App1,host1
+2024-07-16 12:06:00,ERROR,Database connection failed,App1,host1
+2024-07-16 12:07:00,INFO,Application shutdown initiated,App1,host1
+2024-07-16 12:08:00,INFO,Application shutdown completed,App1,host1
diff --git a/...rted/fluent-bit/opensearch_dashboards.yml → ...getting-started/opensearch_dashboards.yml b/...rted/fluent-bit/opensearch_dashboards.yml → ...getting-started/opensearch_dashboards.yml
diff --git a/...ents/getting_started/getting_started_artifacts/csv_file/info/Getting-Started.md b/...ents/getting_started/getting_started_artifacts/csv_file/info/Getting-Started.md
@@ -120,7 +120,7 @@ Under the `getting-started` section you can examine a live docker-compose sample
     |-- logs.csv
 
 ```
-Use the [docker-compose](../getting-started/fluent-bit/docker-complete.yml) you can find a complete:
+Use the [docker-compose](../getting-started/docker-complete.yml) you can find a complete:
 
 `docker compose -f docker-complete.yml up -d` would instantiate the services and start sending the csv sample logs into an index. 
 

diff --git a/...mponents/getting_started/getting_started_artifacts/golang_client/golang_client-1.0.0.json b/...mponents/getting_started/getting_started_artifacts/golang_client/golang_client-1.0.0.json
@@ -11,7 +11,7 @@
     "golang"
   ],
   "author": "OpenSearch",
-  "sourceUrl": "https://github.com/opensearch-project/dashboards-observability/tree/main/server/adaptors/integrations/__data__/repository/golang_client/info",
+  "sourceUrl": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client",
   "workflows": [
     {
       "name": "dashboards",
@@ -84,41 +84,41 @@
                 "type": "host-name"
               }
             ],
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/.env"],
-            "content": "wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/.env",
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/.env"],
+            "content": "wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/.env",
             "description": "Setup docker-compose env variables "
           },
           {
             "name": "Setup fluent-bit folder",
             "label": "Environment Parameters",
             "phase": "docker",
             "type": "file-url",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf",
-              "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/fluent-bit/parsers.conf"
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf",
+              "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/fluent-bit/parsers.conf"
             ],
-            "content": "wget https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf \\\n   https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/fluent-bit/parsers.conf\n",
+            "content": "mkdir -p fluent-bit\nwget -P fluent-bit https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/fluent-bit/fluent-bit.conf \\\n   https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/fluent-bit/parsers.conf\n",
             "description": "Setup docker-compose fluent-bit's service configuration "
           },
           {
             "name": "Setup golang app folder",
             "label": "Environment Parameters",
             "phase": "docker",
             "type": "file-url",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/Dockerfile",
-              "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/go.mod",
-              "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/main.go"
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/golang-app/Dockerfile",
+              "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/golang-app/go.mod",
+              "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/golang-app/main.go"
             ],
-            "content": "wget https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/Dockerfile \\\n https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/go.mod \\\n https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/golang-app/main.go \n",
+            "content": "mkdir -p golang-app\nwget -P golang-app  https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/golang-app/Dockerfile \\\n https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/golang-app/go.mod \\\n https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/golang-app/main.go \n",
             "description": "Setup docker-compose golan-app service configuration "
           },
           {
             "name": "Run docker-compose",
             "label": "live container",
             "type": "file-url",
             "phase": "docker",
-            "info": ["https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/getting-started/docker-compose.yml"],
+            "info": ["https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/getting-started/docker-compose.yml"],
             "description": "Run docker-compose for the gloang-client live example and see the generated index",
-            "content": "wget -O docker-compose.yml https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/docker-compose.yml && docker-compose up --build \n"
+            "content": "wget -O docker-compose.yml https://raw.githubusercontent.com/opensearch-project/opensearch-catalog/main/integrations/observability/golang_client/getting-started/docker-compose.yml && docker compose up --build -d golang-app fluent-bit \n"
           }
         ]
       },
@@ -226,7 +226,7 @@
       ],
       "content": "PUT _index_template/application_logs_template",
       "description": "Setup applicative logs index template in the dev console",
-      "index-template": "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/schemas/applicative-logs-1.0.0.mapping.json",
+      "index-template": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/schemas/applicative-logs-1.0.0.mapping.json",
       "index-pattern-name": "applicative_logs-*"
     }
   ],
@@ -238,7 +238,7 @@
       "https://opensearch.org/docs/latest/dashboards/management/index-patterns/"
     ],
     "description": "Import index patterns `.ndjson` file to the saved objects",
-    "index-pattern": "https://github.com/opensearch-project/opensearch-catalog/tree/main/integrations/observability/golang_client/assets/tutorial-1.0.0.ndjson",
+    "index-pattern": "https://github.com/opensearch-project/opensearch-catalog/blob/main/integrations/observability/golang_client/assets/tutorial-1.0.0.ndjson",
     "index-patterns-name": [
       "applicative_logs-*"
     ]