From b4d589c34fbd59679eb8d183f6ed3181e62d57c9 Mon Sep 17 00:00:00 2001
From: YANGDB <yang.db.dev@gmail.com>
Date: Mon, 14 Aug 2023 13:25:18 -0700
Subject: [PATCH] add docker-compose.yml testing and readme for integration to
 2.9 (#917)

* add docker-compose.yml testing and readme for integration to 2.9

Signed-off-by: YANGDB <yang.db.dev@gmail.com>

* move the screenshots down to the lower part of the README.md file

Signed-off-by: YANGDB <yang.db.dev@gmail.com>

---------

Signed-off-by: YANGDB <yang.db.dev@gmail.com>
---
 .env               |  3 ++
 DEVELOPER_GUIDE.md | 24 ++++++++--------
 README.md          | 50 +++++++++++++++++++++++++++------
 Using-Docker.md    | 47 +++++++++++++++++++++++++++++++
 docker-compose.yml | 69 ++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 172 insertions(+), 21 deletions(-)
 create mode 100644 .env
 create mode 100644 Using-Docker.md
 create mode 100644 docker-compose.yml

diff --git a/.env b/.env
new file mode 100644
index 0000000000..997507e85d
--- /dev/null
+++ b/.env
@@ -0,0 +1,3 @@
+# version for opensearch & opensearch-dashboards docker image
+VERSION=2.9.0
+
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index 8093ce5764..f917628f77 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -4,16 +4,10 @@ So you want to contribute code to this project? Excellent! We're glad you're her
 
 ### Setup
 
-1. Download OpenSearch for the version that matches the [OpenSearch Dashboards version specified in opensearch_dashboards.json](./dashboards-observability/opensearch_dashboards.json#L4) from [opensearch.org](https://opensearch.org/downloads.html).
-1. Download the OpenSearch Dashboards source code for the [version specified in opensearch_dashboards.json](./dashboards-observability/opensearch_dashboards.json#L4) you want to set up.
-1. Change your node version to the version specified in `.node-version` inside the OpenSearch Dashboards root directory.
-1. cd into `OpenSearch-Dashboards` and remove the `plugins` directory.
-1. Check out this package from version control as the `plugins/dashboards-observability` directory.
-```bash
-git clone https://github.com/opensearch-project/dashboards-observability plugins
-git checkout main
-```
-6. Run `yarn osd bootstrap` inside `OpenSearch-Dashboards`.
+1. Git clone OpenSearch-Dashboard for the version that matches the version you want to use [OpenSearch-Dashboards](https://github.com/opensearch-project/OpenSearch-Dashboards)
+2. Change your node version to the version specified in `.node-version` inside the OpenSearch Dashboards root directory. (using [nvm](https://github.com/nvm-sh/nvm) can help for that)
+3. Change directory into `OpenSearch-Dashboards` and git Clone the [Observability-Dashboard](https://github.com/opensearch-project/dashboards-observability) into the `plugins/` directory.
+4. Run `yarn osd bootstrap` inside `OpenSearch-Dashboards`.
 
 Ultimately, your directory structure should look like this:
 
@@ -28,11 +22,15 @@ Ultimately, your directory structure should look like this:
 
 To build the plugin's distributable zip simply run `yarn build`.
 
-Example output: `./build/observability*.zip`
+The output file : `./build/observabilityDashboards-?.?.?.zip` (`?.?.?` stands for the version number)
+
+### Run Directly
+
+Cd back to `OpenSearch-Dashboards` directory and run `yarn start` to start OpenSearch Dashboards including this plugin. OpenSearch Dashboards will be available on `localhost:5601`.
 
-### Run
+### Run Docker
 
-cd back to `OpenSearch-Dashboards` directory and run `yarn start` to start OpenSearch Dashboards including this plugin. OpenSearch Dashboards will be available on `localhost:5601`.
+Run `docker-compose up -d` after changing the [Dockerfile](Dockerfile) as described in this [tutorial](Using-Docker.md)
 
 ### Submitting Changes
 
diff --git a/README.md b/README.md
index eba1a7f066..829e6e6af8 100644
--- a/README.md
+++ b/README.md
@@ -75,8 +75,6 @@ Observability is collection of plugins and applications that let you visualize d
 
 ## Plugin Components
 
-The Dashboards Observability plugin has four components: Trace Analytics, Event Analytics, Observability Dashboards, and Notebooks.
-
 ### Trace Analytics
 
 Trace Analytics page provides instant on dashboards in OpenSearch Dashboards for users to quickly analyze their logs. The plugin uses aggregated results from two indices, `otel-v1-apm-span-*` and `otel-v1-apm-service-map*` created by the otel-trace-raw-processor and service-map-processor, and renders three main views:
@@ -93,21 +91,53 @@ Additionally the fields can be sorted and filtered.
 
 Event Analytics allows user to monitor, correlate, analyze and visualize machine generated data through [Piped Processing Language](https://opensearch.org/docs/latest/observability-plugins/ppl/index/). It also enables the user to turn data-driven events into visualizations and save frequently used ones for quick access.
 
-### Observability Dashboards
+### Metrics Analytics
+
+Metrics Analytics allows ingesting and visualize metric data from log data aggregated within OpenSearch, allowing you to analyze and correlate data across logs, traces, and metrics.
+Previously, you could ingest and visualize only logs and traces from your monitored environments. Using Metrics Analytics one can observe your digital assets with more granularity, gain deeper insight into the health of your infrastructure, and better inform your root cause analysis.
+
+### Operational Panels
 
-Observability Dashboards provides the users to create and view different visualizations on ingested observability data, using Piped Processing Language queries. Use PPL 'where clauses' and datetime timespans to filter all visualizations in the dashboard.
+Operational panels provides the users to create and view different visualizations on ingested observability data, using Piped Processing Language queries. Use PPL 'where clauses' and datetime timespans to filter all visualizations in the panel.
 
 ### Notebooks
 
 Dashboards offer a solution for a few selected use cases, and are great tools if you’re focused on monitoring a known set of metrics over time. Notebooks enables contextual use of data with detailed explanations by allowing a user to combine saved visualizations, text, graphs and decorate data with other reference data sources.
 
-## Documentation
+### Integrations
+Integration is a new type of logical component that allows high level composition of multiple Dashboards / Applications / Queries and more. Integrations can be used to bring together all the metrics and logs from the infrastructure and gain insight into the unified system as a whole.
+
+---
+
+Integration usually consist of the following assets:
+- dashboards & visualization 
+- schema configuration (mapping & templates)
+- data on-boarding (sample data)
+
+[See Integration Documentation](https://github.com/opensearch-project/dashboards-observability/wiki/Integration-Documentation-Reference)
+
+[See Integration Tutorial](https://github.com/opensearch-project/dashboards-observability/wiki/Integration-Creation-Guide)
+
+Please See additional information [documentation](https://opensearch.org/docs/latest/integrations/index)
 
+> Integration Spec [RFC](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3412#schema-support-for-observability)
+
+### Observability Correlation
+In order to be able to correlate information across different signal (represented in different indices) we introduced the notion of correlation into the schema. This information is represented explicitly in both the declarative schema file and the physical mapping file.
+Using this metadata information will enable the knowledge to be projected and allow for analytic engine to produce a join query that will take advantage of these relationships.
+
+The Observability dashboard vision of [telemetry Data Correlation](https://github.com/opensearch-project/dashboards-observability/wiki/Observability-Future-Vision)
+
+---
+## WIKI Home
+Please see our WIKI page for additional information [WIKI](https://opensearch.org/docs/latest/observability/index/) to learn more about our features.
+
+## Documentation
 Please see our technical [documentation](https://opensearch.org/docs/latest/observability/index/) to learn more about its features.
 
 ## Contributing
-
-See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md).
+- See [developer guide](DEVELOPER_GUIDE.md) and [how to contribute to this project](CONTRIBUTING.md).
+- See [Integration Contribution Guide](https://github.com/opensearch-project/dashboards-observability/wiki/Integration-Creation-Guide) for integration creation additional context
 
 ## Getting Help
 
@@ -129,4 +159,8 @@ This project is licensed under the [Apache v2.0 License](LICENSE).
 
 ## Copyright
 
-Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
\ No newline at end of file
+Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details.
+
+## Screenshots
+
+<video src='https://github.com/opensearch-project/dashboards-observability/assets/48943349/8c918fce-35b8-4824-ae2d-6881aad7b9a2' width=180/>
diff --git a/Using-Docker.md b/Using-Docker.md
new file mode 100644
index 0000000000..ec1c9c21f0
--- /dev/null
+++ b/Using-Docker.md
@@ -0,0 +1,47 @@
+# Using Docker
+Using docker as the runtime environment for testing and validating new developments is now available using the next commands:
+
+## Build the plugin
+First the plugin must be build using the `yarn build` command.
+
+Once this build was completed - the expected zip location of the plugin is `./build/observabilityDashboards-?.?.?.zip` where as the `?.?.?` represents the version of this dashboard plugin.
+
+> Note: that the plugin version must correspond to the OpenSearch-Dashboards version - this information appears [here](opensearch_dashboards.json)
+> 
+
+Once the build is completed, make sure to overide the [Dockerfile](Dockerfile) target zip file with the exact name 
+```
+# Install updated plugin
+RUN /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/observabilityDashboards-3.0.0.zip
+```
+
+## Docker imageVersion
+The docker images used by this file are all referencing the [.env](.env) environment file that contains the version that needed to be changed to match your own system. 
+
+## Run the docker image build
+To build the docker image use the next command:
+> `docker build --build-arg VERSION=$(grep VERSION .env | cut -d '=' -f2) -t your_image_name .`
+
+## Run the docker compose
+The [docker-compose](docker-compose.yml) file represents a simple assembly of an OpenSearch cluster with two nodes and an opensearch dashboard that has the updated image with the latest changes in this plugin.
+> This is a test only docker compose that should not be used for production purpose - for such use cases please review this [link](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/)
+
+### Option 1 (All from docker)
+run `docker compose up -d` to start the services and once the service is up and running you can start testing the changes.
+
+### Option 2 (Combined Docker & Dashboard)
+run `docker compose up -d opensearch` to only run the OpenSearch engine - in this case the dashboard has to be run manualy using `yarn start --no-base-path` command in the root dashboards path `./OpenSearch-Dashboards/`
+
+> Note that the OpenSearch version also must correspond to the OpenSearch-Dashboards version
+
+## Accessing the Dashboard
+The dashboard service uses port `localhost:5601` for access and this was already exported in the docker-compose service definition
+```yaml
+    ports:
+      - 5601:5601 # Map host port 5601 to container port 5601
+```
+
+## Security Notice
+There is no security plugin and authentication definitions for this development test demo - pay attention not to use this configuration in a production or any environment that may contain
+confident or personal information without first changing the security definition for accessing the servers - for production use cases please review this [link](https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/)
+
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000000..7d207dfbae
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,69 @@
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+version: '3.9'
+x-default-logging: &logging
+  driver: "json-file"
+  options:
+    max-size: "5m"
+    max-file: "2"
+
+volumes:
+  opensearch-data:
+
+networks:
+  default:
+    name: opensearch-dashboards-demo
+    driver: bridge
+
+services:
+  # OpenSearch store - node (not for production - no security - only for test purpose )
+  opensearch:
+    image: opensearchstaging/opensearch:${VERSION}
+    container_name: opensearch
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch
+      - discovery.seed_hosts=opensearch
+      - cluster.initial_cluster_manager_nodes=opensearch
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
+    ports:
+      - 9200:9200
+      - 9600:9600
+    expose:
+      - "9200"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"]
+      interval: 20s
+      timeout: 10s
+      retries: 10
+
+  # OpenSearch store - dashboard
+  opensearch-dashboards:
+    container_name: opensearch-dashboards
+    build:
+      context: ./
+      dockerfile: Dockerfile
+      args:
+        - VERSION=${VERSION}
+
+    ports:
+      - 5601:5601 # Map host port 5601 to container port 5601
+    expose:
+      - "5601" # Expose port 5601 for web access to OpenSearch Dashboards
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query
+    depends_on:
+      - opensearch
+