Merge pull request #195 from nightly-labs/finish-grafana-setup

Finish grafana setup

.env

@@ -2,6 +2,10 @@ ENV=DEV # PROD or DEV
 ONLY_RELAY_SERVICE=FALSE # TRUE - This will start only nightly relay service without cloud service
 NONCE=VERY_SECRET_NONCE
 MAILER_ADDRESS=[email protected]
+# Different than db address specified in infra/.env due to docker shenanigans, used in setting up datasource in grafana
+# This is a address of the whole docker interface not just container with database
+# Can be found by looking for docker0 entry by sing command ifconfig/ip -a
+DATABASE_ADDRESS=172.17.0.1
 GRAFANA_BASE_PATH=http://localhost:3005/api
 # TEST LOGIN DO NO USE IN PRODUCTION
 GF_SECURITY_ADMIN_USER=admin
@@ -10,6 +14,7 @@ GF_SECURITY_ADMIN_PASSWORD=admin
 # TEST PASSWORD DO NO USE IN PRODUCTION
 MAILER_PASSWORD=ZtA5gFKMsXzHmEm
 MAILER_ACTIVE=FALSE
+
 # Generated so it can work with grafana 
 # ssh-keygen -t rsa -b 4096 -m PEM -f grafana.key -N ""
 # openssl rsa -in grafana.key -pubout -outform PEM -out grafana.key.pub

infra/.env

@@ -12,6 +12,10 @@ POSTGRES_PASSWORD=password12345
 POSTGRES_DB=connect_db
 PG_DATA=/home/postgres/pgdata
 
+# Grafana read only database user
+GRAFANA_DB_USERNAME=grafanaaccess
+GRAFANA_DB_PASSWORD=very-stronk-password 
+
 # Images
 # https://github.com/timescale/timescaledb-docker-ha
 TIMESCALEDB_IMAGE=timescale/timescaledb-ha:pg15-ts2.10

infra/scripts/clean_start.sh

@@ -227,6 +227,47 @@ if wait_for_db_ready; then
   #   echo "Not a CI, displaying the container logs..."
   #   docker logs -f $CONTAINER_ID
   # fi
+
+  echo "Creating a restricted user for Grafana in the database..."
+
+  # Verify the variables are set
+  if [ -z "$GRAFANA_DB_USERNAME" ] || [ -z "$GRAFANA_DB_PASSWORD" ]; then
+    echo "Error: GRAFANA_DB_USERNAME or GRAFANA_DB_PASSWORD is not set. Please set these variables."
+    exit 1
+  fi
+  printf "DATABASE NAME: $POSTGRES_DB\n"
+
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "CREATE USER $GRAFANA_DB_USERNAME WITH PASSWORD '$GRAFANA_DB_PASSWORD';"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "GRANT CONNECT ON DATABASE $POSTGRES_DB TO $GRAFANA_DB_USERNAME;"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "GRANT USAGE ON SCHEMA public TO $GRAFANA_DB_USERNAME;"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO $GRAFANA_DB_USERNAME;"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $GRAFANA_DB_USERNAME;"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "REVOKE DELETE ON ALL TABLES IN SCHEMA public FROM $GRAFANA_DB_USERNAME;"
+  docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE DELETE ON TABLES FROM $GRAFANA_DB_USERNAME;"
+
+  echo "Restricted user for Grafana has been created with SELECT privileges only."
+
+  # Check if the user was created successfully
+  echo "Verifying the user creation and connection..."
+  # This query will return a row if the user exists
+  user_exists=$(docker exec -u postgres $CONTAINER_ID psql -d "$POSTGRES_DB" -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$GRAFANA_DB_USERNAME';")
+
+  if [[ "$user_exists" == "1" ]]; then
+    echo "User $GRAFANA_DB_USERNAME exists in the database."
+  else
+    echo "User $GRAFANA_DB_USERNAME does not exist. Please check the creation process."
+    exit 1
+  fi
+
+  # Check if the new user can connect and run a query
+  docker exec -u postgres $CONTAINER_ID psql -U "$GRAFANA_DB_USERNAME" -d "$POSTGRES_DB" -c "SELECT 1;" &>/dev/null
+
+  if [ $? -eq 0 ]; then
+    echo "User $GRAFANA_DB_USERNAME created and verified successfully."
+  else
+    echo "Failed to verify user $GRAFANA_DB_USERNAME. Please check the PostgreSQL logs."
+    exit 1
+  fi
 else
   echo "Failed to confirm TimescaleDB readiness after restart. Check logs for more details."
   exit 1

server/src/env.rs

@@ -10,6 +10,7 @@ pub struct ENV {
     pub NONCE: String,
     pub MAILER_ADDRESS: String,
     pub MAILER_PASSWORD: String,
+    pub DATABASE_ADDRESS: String,
     pub GRAFANA_BASE_PATH: String,
     pub GF_SECURITY_ADMIN_USER: String,
     pub GF_SECURITY_ADMIN_PASSWORD: String,
@@ -35,6 +36,8 @@ pub fn get_env() -> &'static ENV {
                 .expect("Failed to get MAILER_ADDRESS env"),
             MAILER_PASSWORD: std::env::var("MAILER_PASSWORD")
                 .expect("Failed to get MAILER_PASSWORD env"),
+            DATABASE_ADDRESS: std::env::var("DATABASE_ADDRESS")
+                .expect("Failed to get DATABASE_ADDRESS env"),
             GRAFANA_BASE_PATH: std::env::var("GRAFANA_BASE_PATH")
                 .expect("Failed to get GRAFANA_BASE_PATH env"),
             GF_SECURITY_ADMIN_USER: std::env::var("GF_SECURITY_ADMIN_USER")
@@ -85,3 +88,6 @@ pub fn GF_SECURITY_ADMIN_PASSWORD() -> &'static str {
 pub fn MAILER_ACTIVE() -> bool {
     get_env().MAILER_ACTIVE
 }
+pub fn DATABASE_ADDRESS() -> &'static str {
+    get_env().DATABASE_ADDRESS.as_str()
+}

server/src/http/cloud/grafana_utils/import_template_dashboard.rs

@@ -1,14 +1,15 @@
 use crate::{
-    statics::{DASHBOARD_TEMPLATE_UID, TEMPLATES_FOLDER_UID},
+    statics::{DASHBOARD_TEMPLATE_UID, POSTGRES_DATASOURCE_UID, TEMPLATES_FOLDER_UID},
     structs::cloud::grafana_error::handle_grafana_error,
 };
 use axum::http::StatusCode;
+use log::{error, info, warn};
 use openapi::{
     apis::{
         configuration::Configuration,
         dashboards_api::{get_dashboard_by_uid, import_dashboard},
     },
-    models::ImportDashboardRequest,
+    models::{ImportDashboardInput, ImportDashboardRequest},
 };
 use serde_json::Value;
 use std::{env, sync::Arc};
@@ -34,44 +35,56 @@ pub async fn setup_templates_dashboard(
         Ok(response) => match response.dashboard {
             Some(_dashboard) => return Ok(()),
             None => {
-                // Try to import the dashboard
+                warn!("Failed to get dashboard data event though grafana returned 200");
+
+                // Try to import the dashboard anyway
                 let request = ImportDashboardRequest {
                     dashboard: Some(dashboard),
                     folder_id: None,
                     folder_uid: Some(TEMPLATES_FOLDER_UID.to_string()),
-                    overwrite: Some(false),
-                    inputs: None,
+                    overwrite: Some(true),
+                    inputs: Some(vec![ImportDashboardInput {
+                        name: Some("DS_GRAFANA-POSTGRESQL-DATASOURCE".to_string()),
+                        plugin_id: Some("grafana-postgres-datasource".to_string()),
+                        r#type: Some("datasource".to_string()),
+                        value: Some(POSTGRES_DATASOURCE_UID.to_string()),
+                    }]),
                     path: None,
                     plugin_id: None,
                 };
 
                 match import_dashboard(&grafana_conf, request).await {
                     Ok(_) => return Ok(()),
                     Err(err) => {
-                        println!("Failed to import template dashboard: {:?}", err);
+                        error!("Failed to import template dashboard: {:?}", err);
                         return Err(handle_grafana_error(err));
                     }
                 }
             }
         },
-        Err(err) => {
-            println!("Failed to import template dashboard: {:?}", err);
+        Err(_) => {
+            info!("Template dashboard does not exists, creating it");
 
-            // Try to import the dashboard anyway
+            // Try to import the dashboard
             let request = ImportDashboardRequest {
                 dashboard: Some(dashboard),
                 folder_id: None,
                 folder_uid: Some(TEMPLATES_FOLDER_UID.to_string()),
-                overwrite: Some(false),
-                inputs: None,
+                overwrite: Some(true),
+                inputs: Some(vec![ImportDashboardInput {
+                    name: Some("DS_GRAFANA-POSTGRESQL-DATASOURCE".to_string()),
+                    plugin_id: Some("grafana-postgres-datasource".to_string()),
+                    r#type: Some("datasource".to_string()),
+                    value: Some(POSTGRES_DATASOURCE_UID.to_string()),
+                }]),
                 path: None,
                 plugin_id: None,
             };
 
             match import_dashboard(&grafana_conf, request).await {
                 Ok(_) => return Ok(()),
                 Err(err) => {
-                    println!("Failed to import template dashboard: {:?}", err);
+                    error!("Failed to import template dashboard: {:?}", err);
                     return Err(handle_grafana_error(err));
                 }
             }

server/src/http/cloud/grafana_utils/mod.rs

@@ -3,4 +3,5 @@ pub mod create_new_app;
 pub mod create_new_team;
 pub mod import_template_dashboard;
 pub mod remove_user_from_the_team;
+pub mod setup_database_datasource;
 pub mod setup_template_folder;

server/src/http/cloud/grafana_utils/setup_database_datasource.rs

@@ -0,0 +1,49 @@
+use crate::{
+    env::DATABASE_ADDRESS,
+    infra_env::{GRAFANA_DB_USERNAME, POSTGRES_DB},
+    statics::POSTGRES_DATASOURCE_UID,
+    structs::cloud::grafana_error::handle_grafana_error,
+};
+use axum::http::StatusCode;
+use openapi::{
+    apis::{
+        configuration::Configuration,
+        datasources_api::{add_data_source, get_data_source_by_uid},
+    },
+    models::AddDataSourceCommand,
+};
+use std::sync::Arc;
+
+pub async fn setup_database_datasource(
+    grafana_conf: &Arc<Configuration>,
+) -> Result<(), (StatusCode, String)> {
+    // Check if datasource already exists,  otherwise create it
+    if let Err(_) = get_data_source_by_uid(grafana_conf, POSTGRES_DATASOURCE_UID).await {
+        let request_payload = AddDataSourceCommand {
+            name: Some("Postgres".to_string()),
+            r#type: Some("postgres".to_string()),
+            access: Some("proxy".to_string()),
+            // DATABASE ADDRESS from main env file
+            url: Some(DATABASE_ADDRESS().to_string()),
+            database: Some(POSTGRES_DB().to_string()),
+            user: Some(GRAFANA_DB_USERNAME().to_string()),
+            basic_auth: None,
+            with_credentials: Some(false),
+            is_default: Some(true),
+            json_data: None,
+            uid: Some(POSTGRES_DATASOURCE_UID.to_string()),
+            basic_auth_user: None,
+            secure_json_data: None,
+        };
+
+        match add_data_source(&grafana_conf, request_payload).await {
+            Ok(_) => return Ok(()),
+            Err(err) => {
+                println!("Failed to import database datasource: {:?}", err);
+                return Err(handle_grafana_error(err));
+            }
+        }
+    }
+
+    Ok(())
+}

server/src/http/cloud/grafana_utils/setup_template_folder.rs

@@ -1,5 +1,6 @@
 use crate::{statics::TEMPLATES_FOLDER_UID, structs::cloud::grafana_error::handle_grafana_error};
 use axum::http::StatusCode;
+use log::info;
 use openapi::{
     apis::{
         configuration::Configuration,
@@ -34,8 +35,8 @@ pub async fn setup_templates_folder(
                 }
             }
         },
-        Err(err) => {
-            println!("Failed to get templates folder: {:?}", err);
+        Err(_) => {
+            info!("Templates folder does not exist, creating it");
 
             // Try to create the folder anyway
             let folder_request = CreateFolderCommand {