From 5979439d1b30aa3fec25a396134a1290786f23b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CGiems=E2=80=9D?= <“hubert.wabia@gmail.com”> Date: Fri, 9 Aug 2024 14:33:56 +0200 Subject: [PATCH] grafana init setup --- .env | 4 + server/src/env.rs | 6 + .../import_template_dashboard.rs | 37 ++-- server/src/http/cloud/grafana_utils/mod.rs | 1 + .../setup_database_datasource.rs | 49 +++++ .../grafana_utils/setup_template_folder.rs | 5 +- server/src/infra_env.rs | 168 ++++++++++++++++++ server/src/lib.rs | 1 + server/src/mailer/request_handler/utils.rs | 32 ++-- server/src/statics.rs | 1 + server/src/utils.rs | 12 +- 11 files changed, 282 insertions(+), 34 deletions(-) create mode 100644 server/src/http/cloud/grafana_utils/setup_database_datasource.rs create mode 100644 server/src/infra_env.rs diff --git a/.env b/.env index c3025712..6d2e50db 100644 --- a/.env +++ b/.env @@ -2,6 +2,10 @@ ENV=DEV # PROD or DEV ONLY_RELAY_SERVICE=FALSE # TRUE - This will start only nightly relay service without cloud service NONCE=VERY_SECRET_NONCE MAILER_ADDRESS=do_not_reply@nightly.app +# Different than db address specified in infra/.env due to docker shenanigans, used in setting up datasource in grafana +# This is a address of the whole docker interface not just container with database +# Can be found by looking for docker0 entry by sing command ifconfig/ip -a +DATABASE_ADDRESS=172.17.0.1 GRAFANA_BASE_PATH=http://localhost:3005/api # TEST LOGIN DO NO USE IN PRODUCTION GF_SECURITY_ADMIN_USER=admin diff --git a/server/src/env.rs b/server/src/env.rs index 11f9a052..cbcc9913 100644 --- a/server/src/env.rs +++ b/server/src/env.rs @@ -10,6 +10,7 @@ pub struct ENV { pub NONCE: String, pub MAILER_ADDRESS: String, pub MAILER_PASSWORD: String, + pub DATABASE_ADDRESS: String, pub GRAFANA_BASE_PATH: String, pub GF_SECURITY_ADMIN_USER: String, pub GF_SECURITY_ADMIN_PASSWORD: String, @@ -35,6 +36,8 @@ pub fn get_env() -> &'static ENV { .expect("Failed to get MAILER_ADDRESS env"), MAILER_PASSWORD: std::env::var("MAILER_PASSWORD") .expect("Failed to get MAILER_PASSWORD env"), + DATABASE_ADDRESS: std::env::var("DATABASE_ADDRESS") + .expect("Failed to get DATABASE_ADDRESS env"), GRAFANA_BASE_PATH: std::env::var("GRAFANA_BASE_PATH") .expect("Failed to get GRAFANA_BASE_PATH env"), GF_SECURITY_ADMIN_USER: std::env::var("GF_SECURITY_ADMIN_USER") @@ -85,3 +88,6 @@ pub fn GF_SECURITY_ADMIN_PASSWORD() -> &'static str { pub fn MAILER_ACTIVE() -> bool { get_env().MAILER_ACTIVE } +pub fn DATABASE_ADDRESS() -> &'static str { + get_env().DATABASE_ADDRESS.as_str() +} diff --git a/server/src/http/cloud/grafana_utils/import_template_dashboard.rs b/server/src/http/cloud/grafana_utils/import_template_dashboard.rs index 9c2fb451..858cb038 100644 --- a/server/src/http/cloud/grafana_utils/import_template_dashboard.rs +++ b/server/src/http/cloud/grafana_utils/import_template_dashboard.rs @@ -1,14 +1,15 @@ use crate::{ - statics::{DASHBOARD_TEMPLATE_UID, TEMPLATES_FOLDER_UID}, + statics::{DASHBOARD_TEMPLATE_UID, POSTGRES_DATASOURCE_UID, TEMPLATES_FOLDER_UID}, structs::cloud::grafana_error::handle_grafana_error, }; use axum::http::StatusCode; +use log::{error, info, warn}; use openapi::{ apis::{ configuration::Configuration, dashboards_api::{get_dashboard_by_uid, import_dashboard}, }, - models::ImportDashboardRequest, + models::{ImportDashboardInput, ImportDashboardRequest}, }; use serde_json::Value; use std::{env, sync::Arc}; @@ -34,13 +35,20 @@ pub async fn setup_templates_dashboard( Ok(response) => match response.dashboard { Some(_dashboard) => return Ok(()), None => { - // Try to import the dashboard + warn!("Failed to get dashboard data event though grafana returned 200"); + + // Try to import the dashboard anyway let request = ImportDashboardRequest { dashboard: Some(dashboard), folder_id: None, folder_uid: Some(TEMPLATES_FOLDER_UID.to_string()), - overwrite: Some(false), - inputs: None, + overwrite: Some(true), + inputs: Some(vec![ImportDashboardInput { + name: Some("DS_GRAFANA-POSTGRESQL-DATASOURCE".to_string()), + plugin_id: Some("grafana-postgres-datasource".to_string()), + r#type: Some("datasource".to_string()), + value: Some(POSTGRES_DATASOURCE_UID.to_string()), + }]), path: None, plugin_id: None, }; @@ -48,22 +56,27 @@ pub async fn setup_templates_dashboard( match import_dashboard(&grafana_conf, request).await { Ok(_) => return Ok(()), Err(err) => { - println!("Failed to import template dashboard: {:?}", err); + error!("Failed to import template dashboard: {:?}", err); return Err(handle_grafana_error(err)); } } } }, - Err(err) => { - println!("Failed to import template dashboard: {:?}", err); + Err(_) => { + info!("Template dashboard does not exists, creating it"); - // Try to import the dashboard anyway + // Try to import the dashboard let request = ImportDashboardRequest { dashboard: Some(dashboard), folder_id: None, folder_uid: Some(TEMPLATES_FOLDER_UID.to_string()), - overwrite: Some(false), - inputs: None, + overwrite: Some(true), + inputs: Some(vec![ImportDashboardInput { + name: Some("DS_GRAFANA-POSTGRESQL-DATASOURCE".to_string()), + plugin_id: Some("grafana-postgres-datasource".to_string()), + r#type: Some("datasource".to_string()), + value: Some(POSTGRES_DATASOURCE_UID.to_string()), + }]), path: None, plugin_id: None, }; @@ -71,7 +84,7 @@ pub async fn setup_templates_dashboard( match import_dashboard(&grafana_conf, request).await { Ok(_) => return Ok(()), Err(err) => { - println!("Failed to import template dashboard: {:?}", err); + error!("Failed to import template dashboard: {:?}", err); return Err(handle_grafana_error(err)); } } diff --git a/server/src/http/cloud/grafana_utils/mod.rs b/server/src/http/cloud/grafana_utils/mod.rs index 6bad89f7..8223494b 100644 --- a/server/src/http/cloud/grafana_utils/mod.rs +++ b/server/src/http/cloud/grafana_utils/mod.rs @@ -3,4 +3,5 @@ pub mod create_new_app; pub mod create_new_team; pub mod import_template_dashboard; pub mod remove_user_from_the_team; +pub mod setup_database_datasource; pub mod setup_template_folder; diff --git a/server/src/http/cloud/grafana_utils/setup_database_datasource.rs b/server/src/http/cloud/grafana_utils/setup_database_datasource.rs new file mode 100644 index 00000000..220c4239 --- /dev/null +++ b/server/src/http/cloud/grafana_utils/setup_database_datasource.rs @@ -0,0 +1,49 @@ +use crate::{ + env::DATABASE_ADDRESS, + infra_env::{POSTGRES_DB, POSTGRES_USER}, + statics::POSTGRES_DATASOURCE_UID, + structs::cloud::grafana_error::handle_grafana_error, +}; +use axum::http::StatusCode; +use openapi::{ + apis::{ + configuration::Configuration, + datasources_api::{add_data_source, get_data_source_by_uid}, + }, + models::AddDataSourceCommand, +}; +use std::sync::Arc; + +pub async fn setup_database_datasource( + grafana_conf: &Arc, +) -> Result<(), (StatusCode, String)> { + // Check if datasource already exists, otherwise create it + if let Err(_) = get_data_source_by_uid(grafana_conf, POSTGRES_DATASOURCE_UID).await { + let request_payload = AddDataSourceCommand { + name: Some("Postgres".to_string()), + r#type: Some("postgres".to_string()), + access: Some("proxy".to_string()), + // DATABASE ADDRESS from main env file + url: Some(DATABASE_ADDRESS().to_string()), + database: Some(POSTGRES_DB().to_string()), + user: Some(POSTGRES_USER().to_string()), + basic_auth: None, + with_credentials: Some(false), + is_default: Some(true), + json_data: None, + uid: Some(POSTGRES_DATASOURCE_UID.to_string()), + basic_auth_user: None, + secure_json_data: None, + }; + + match add_data_source(&grafana_conf, request_payload).await { + Ok(_) => return Ok(()), + Err(err) => { + println!("Failed to import database datasource: {:?}", err); + return Err(handle_grafana_error(err)); + } + } + } + + Ok(()) +} diff --git a/server/src/http/cloud/grafana_utils/setup_template_folder.rs b/server/src/http/cloud/grafana_utils/setup_template_folder.rs index daf3a72d..37b385d1 100644 --- a/server/src/http/cloud/grafana_utils/setup_template_folder.rs +++ b/server/src/http/cloud/grafana_utils/setup_template_folder.rs @@ -1,5 +1,6 @@ use crate::{statics::TEMPLATES_FOLDER_UID, structs::cloud::grafana_error::handle_grafana_error}; use axum::http::StatusCode; +use log::info; use openapi::{ apis::{ configuration::Configuration, @@ -34,8 +35,8 @@ pub async fn setup_templates_folder( } } }, - Err(err) => { - println!("Failed to get templates folder: {:?}", err); + Err(_) => { + info!("Templates folder does not exist, creating it"); // Try to create the folder anyway let folder_request = CreateFolderCommand { diff --git a/server/src/infra_env.rs b/server/src/infra_env.rs new file mode 100644 index 00000000..deb74985 --- /dev/null +++ b/server/src/infra_env.rs @@ -0,0 +1,168 @@ +#![allow(non_snake_case)] +use once_cell::sync::OnceCell; + +#[derive(Debug)] +pub struct INFRA_ENV { + pub ENVIRONMENT: String, + pub ONLY_DATABASE: bool, + pub DATABASE_ADDRESS: String, + pub DATABASE_PORT: String, + pub POSTGRES_USER: String, + pub POSTGRES_PASSWORD: String, + pub POSTGRES_DB: String, + pub PG_DATA: String, + pub TIMESCALEDB_IMAGE: String, + pub OFELIA_IMAGE: String, + pub TIMESCALEDB_DATA: String, + pub TIMESCALEDB_BACKUPS: String, + pub TIMESCALEDB_LOGS: String, + pub TIMESCALEDB_PGBACKREST_CONFIG: String, + pub OFELIA_LOGS: String, + pub MANUAL_BACKUPS: String, + pub CUSTOM_ENTRYPOINT: String, + pub OFELIA_SMTP_HOST: String, + pub OFELIA_SMTP_PORT: String, + pub OFELIA_SMTP_USER: String, + pub OFELIA_SMTP_PASSWORD: String, + pub OFELIA_EMAIL_FROM: String, + pub OFELIA_EMAIL_TO: String, +} + +pub fn get_env() -> &'static INFRA_ENV { + static INSTANCE: OnceCell = OnceCell::new(); + + INSTANCE.get_or_init(|| { + dotenvy::from_filename(".env").ok(); + + let env = INFRA_ENV { + ENVIRONMENT: std::env::var("ENV").unwrap_or_else(|_| "DEV".to_string()), // Default to DEV if not set + ONLY_DATABASE: std::env::var("ONLY_DATABASE") + .unwrap_or_else(|_| "false".to_string()) + .eq_ignore_ascii_case("true"), + DATABASE_ADDRESS: std::env::var("DATABASE_ADDRESS") + .expect("Failed to get DATABASE_ADDRESS env"), + DATABASE_PORT: std::env::var("DATABASE_PORT").unwrap_or_else(|_| "5432".to_string()), // Default to 5432 if not set + POSTGRES_USER: std::env::var("POSTGRES_USER").expect("Failed to get POSTGRES_USER env"), + POSTGRES_PASSWORD: std::env::var("POSTGRES_PASSWORD") + .expect("Failed to get POSTGRES_PASSWORD env"), + POSTGRES_DB: std::env::var("POSTGRES_DB").expect("Failed to get POSTGRES_DB env"), + PG_DATA: std::env::var("PG_DATA").expect("Failed to get PG_DATA env"), + TIMESCALEDB_IMAGE: std::env::var("TIMESCALEDB_IMAGE") + .expect("Failed to get TIMESCALEDB_IMAGE env"), + OFELIA_IMAGE: std::env::var("OFELIA_IMAGE").expect("Failed to get OFELIA_IMAGE env"), + TIMESCALEDB_DATA: std::env::var("TIMESCALEDB_DATA") + .expect("Failed to get TIMESCALEDB_DATA env"), + TIMESCALEDB_BACKUPS: std::env::var("TIMESCALEDB_BACKUPS") + .expect("Failed to get TIMESCALEDB_BACKUPS env"), + TIMESCALEDB_LOGS: std::env::var("TIMESCALEDB_LOGS") + .expect("Failed to get TIMESCALEDB_LOGS env"), + TIMESCALEDB_PGBACKREST_CONFIG: std::env::var("TIMESCALEDB_PGBACKREST_CONFIG") + .expect("Failed to get TIMESCALEDB_PGBACKREST_CONFIG env"), + OFELIA_LOGS: std::env::var("OFELIA_LOGS").expect("Failed to get OFELIA_LOGS env"), + MANUAL_BACKUPS: std::env::var("MANUAL_BACKUPS") + .expect("Failed to get MANUAL_BACKUPS env"), + CUSTOM_ENTRYPOINT: std::env::var("CUSTOM_ENTRYPOINT") + .expect("Failed to get CUSTOM_ENTRYPOINT env"), + OFELIA_SMTP_HOST: std::env::var("OFELIA_SMTP_HOST") + .expect("Failed to get OFELIA_SMTP_HOST env"), + OFELIA_SMTP_PORT: std::env::var("OFELIA_SMTP_PORT") + .expect("Failed to get OFELIA_SMTP_PORT env"), + OFELIA_SMTP_USER: std::env::var("OFELIA_SMTP_USER") + .expect("Failed to get OFELIA_SMTP_USER env"), + OFELIA_SMTP_PASSWORD: std::env::var("OFELIA_SMTP_PASSWORD") + .expect("Failed to get OFELIA_SMTP_PASSWORD env"), + OFELIA_EMAIL_FROM: std::env::var("OFELIA_EMAIL_FROM") + .expect("Failed to get OFELIA_EMAIL_FROM env"), + OFELIA_EMAIL_TO: std::env::var("OFELIA_EMAIL_TO") + .expect("Failed to get OFELIA_EMAIL_TO env"), + }; + return env; + }) +} +pub fn ONLY_DATABASE() -> bool { + get_env().ONLY_DATABASE +} + +pub fn DATABASE_ADDRESS() -> &'static str { + get_env().DATABASE_ADDRESS.as_str() +} + +pub fn DATABASE_PORT() -> &'static str { + get_env().DATABASE_PORT.as_str() +} + +pub fn POSTGRES_USER() -> &'static str { + get_env().POSTGRES_USER.as_str() +} + +pub fn POSTGRES_PASSWORD() -> &'static str { + get_env().POSTGRES_PASSWORD.as_str() +} + +pub fn POSTGRES_DB() -> &'static str { + get_env().POSTGRES_DB.as_str() +} + +pub fn PG_DATA() -> &'static str { + get_env().PG_DATA.as_str() +} + +pub fn TIMESCALEDB_IMAGE() -> &'static str { + get_env().TIMESCALEDB_IMAGE.as_str() +} + +pub fn OFELIA_IMAGE() -> &'static str { + get_env().OFELIA_IMAGE.as_str() +} + +pub fn TIMESCALEDB_DATA() -> &'static str { + get_env().TIMESCALEDB_DATA.as_str() +} + +pub fn TIMESCALEDB_BACKUPS() -> &'static str { + get_env().TIMESCALEDB_BACKUPS.as_str() +} + +pub fn TIMESCALEDB_LOGS() -> &'static str { + get_env().TIMESCALEDB_LOGS.as_str() +} + +pub fn TIMESCALEDB_PGBACKREST_CONFIG() -> &'static str { + get_env().TIMESCALEDB_PGBACKREST_CONFIG.as_str() +} + +pub fn OFELIA_LOGS() -> &'static str { + get_env().OFELIA_LOGS.as_str() +} + +pub fn MANUAL_BACKUPS() -> &'static str { + get_env().MANUAL_BACKUPS.as_str() +} + +pub fn CUSTOM_ENTRYPOINT() -> &'static str { + get_env().CUSTOM_ENTRYPOINT.as_str() +} + +pub fn OFELIA_SMTP_HOST() -> &'static str { + get_env().OFELIA_SMTP_HOST.as_str() +} + +pub fn OFELIA_SMTP_PORT() -> &'static str { + get_env().OFELIA_SMTP_PORT.as_str() +} + +pub fn OFELIA_SMTP_USER() -> &'static str { + get_env().OFELIA_SMTP_USER.as_str() +} + +pub fn OFELIA_SMTP_PASSWORD() -> &'static str { + get_env().OFELIA_SMTP_PASSWORD.as_str() +} + +pub fn OFELIA_EMAIL_FROM() -> &'static str { + get_env().OFELIA_EMAIL_FROM.as_str() +} + +pub fn OFELIA_EMAIL_TO() -> &'static str { + get_env().OFELIA_EMAIL_TO.as_str() +} diff --git a/server/src/lib.rs b/server/src/lib.rs index 161755f6..381ac02f 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -4,6 +4,7 @@ pub mod env; pub mod errors; pub mod handle_error; pub mod http; +pub mod infra_env; pub mod ip_geolocation; pub mod mailer; pub mod middlewares; diff --git a/server/src/mailer/request_handler/utils.rs b/server/src/mailer/request_handler/utils.rs index 2142e5c8..422b8ef5 100644 --- a/server/src/mailer/request_handler/utils.rs +++ b/server/src/mailer/request_handler/utils.rs @@ -1,24 +1,22 @@ use anyhow::Result; use lettre::{ - message::{header::ContentType, Mailbox}, - Message, + message::{header::ContentType, Mailbox}, + Message, }; pub fn create_message( - body: String, - mailbox: Mailbox, - receiver: &String, - subject: String, + body: String, + mailbox: Mailbox, + receiver: &String, + subject: String, ) -> Result { - Message::builder() - .from(mailbox) - .to( - format!("Hello <{receiver}>") - .parse() - .map_err(|e| anyhow::anyhow!("Could not parse to: {:?}", e))?, - ) - .subject(subject) - .header(ContentType::TEXT_HTML) - .body(body) - .map_err(|e| anyhow::anyhow!("Could not build email: {:?}", e)) + Message::builder() + .from(mailbox) + .to(format!("Hello <{receiver}>") + .parse() + .map_err(|e| anyhow::anyhow!("Could not parse to: {:?}", e))?) + .subject(subject) + .header(ContentType::TEXT_HTML) + .body(body) + .map_err(|e| anyhow::anyhow!("Could not build email: {:?}", e)) } diff --git a/server/src/statics.rs b/server/src/statics.rs index 78b66b23..8c2952dc 100644 --- a/server/src/statics.rs +++ b/server/src/statics.rs @@ -7,6 +7,7 @@ pub const USERS_AMOUNT_LIMIT_PER_TEAM: usize = 50; pub const DASHBOARD_TEMPLATE_UID: &str = "TEMPLATE_UID"; pub const TEMPLATES_FOLDER_UID: &str = "TEMPLATE_FOLDER_UID"; +pub const POSTGRES_DATASOURCE_UID: &str = "POSTGRES_DATASOURCE_UID"; // Name must be 3-30 characters long and include only alphanumeric characters, underscores, or slashes. pub static NAME_REGEX: Lazy = diff --git a/server/src/utils.rs b/server/src/utils.rs index de8d41d2..a0a42547 100644 --- a/server/src/utils.rs +++ b/server/src/utils.rs @@ -1,6 +1,7 @@ use crate::{ http::cloud::grafana_utils::{ import_template_dashboard::setup_templates_dashboard, + setup_database_datasource::setup_database_datasource, setup_template_folder::setup_templates_folder, }, structs::{wallet_metadata::WalletMetadata, wallets::*}, @@ -53,7 +54,12 @@ pub async fn import_template_dashboards(grafana_client: &Arc) { // Check if folder exists if not create it setup_templates_folder(&grafana_client).await.unwrap(); - // Check if dashboard exists if not create it - // TODO fix, aka setup dynamically datasource before importing - // setup_templates_dashboard(&grafana_client).await.unwrap(); + // Check if database datasource was set up in grafana + setup_database_datasource(&grafana_client).await.unwrap(); + + // Check if template dashboard exists if not create it + setup_templates_dashboard(&grafana_client).await.unwrap(); + + // Setup global dashboard + // TODO }