Merge pull request #180 from nightly-labs/postgress_settings

Postgress settings

.github/workflows/connect-test-local.yml

@@ -33,7 +33,11 @@ jobs:
         run: |
           docker-compose down
           rm -rf .infra/target
-          docker-compose up -d --no-deps --force-recreate --remove-orphans
+          rm -rf .infra/backups
+          rm -rf .infra/config
+          rm -rf .infra/logs
+          rm -rf .infra/ofelia_logs
+          echo ${{secrets.TESTING_LAPTOP_PASSWORD}} | sudo -S ./scripts/clean_start.sh
       - name: Setup Grafana
         working-directory: ./grafana
         run: |
@@ -114,3 +118,7 @@ jobs:
         run: |
           docker-compose down
           rm -rf .infra/target
+          rm -rf .infra/backups
+          rm -rf .infra/config
+          rm -rf .infra/logs
+          rm -rf .infra/ofelia_logs

.gitignore

@@ -1,4 +1,9 @@
 /target
 /Cargo.lock
 /.vscode
-/grafana-client-gen/build
+/grafana-client-gen/build
+/infra/target
+/infra/config
+/infra/logs
+/infra/backups
+/infra/ofelia_logs

infra/.env

@@ -1,5 +1,32 @@
-ENV=DEV # PROD or DEV
-PGDATA=/home/postgres/pgdata/data
+ENV=DEV # Does nothing for now
+
+# Database Configuration
+# Those two has to be the same 
 POSTGRES_USER=admin1234
+PGUSER=admin1234
+# -----------------------
 POSTGRES_PASSWORD=password12345
 POSTGRES_DB=connect_db
+PG_DATA=/home/postgres/pgdata
+
+# Images
+TIMESCALEDB_IMAGE=timescale/timescaledb-ha:pg15-ts2.10
+OFELIA_IMAGE=mcuadros/ofelia:988d988
+
+# Volume Bindings
+TIMESCALEDB_DATA=./target
+TIMESCALEDB_BACKUPS=./backups
+TIMESCALEDB_LOGS=./logs
+TIMESCALEDB_PGBACKREST_CONFIG=./config
+OFELIA_LOGS=./ofelia_logs
+CUSTOM_ENTRYPOINT=./scripts/custom_entrypoint.sh
+
+# Ofelia Configuration
+OFELIA_SMTP_HOST=smtp.example.com
+OFELIA_SMTP_PORT=587
+# Those two has to be the same 
+OFELIA_SMTP_USER=[email protected]
+OFELIA_EMAIL_FROM=[email protected]
+# -----------------------
+OFELIA_SMTP_PASSWORD=examplepassword
+OFELIA_EMAIL_TO=[email protected]

infra/docker-compose.yaml

@@ -1,14 +1,66 @@
 services:
   timescaledb:
-    image: timescale/timescaledb-ha:pg15
+    image: ${TIMESCALEDB_IMAGE}
     ports:
       - 5432:5432
     volumes:
-      - ./target:/var/lib/postgresql/data
+      - ${TIMESCALEDB_DATA}:/home/postgres/pgdata
+      - ${TIMESCALEDB_BACKUPS}:/var/lib/pgbackrest
+      - ${TIMESCALEDB_PGBACKREST_CONFIG}:/home/postgres/pgdata/backup
+      - ${TIMESCALEDB_LOGS}:/var/log
+      - ${CUSTOM_ENTRYPOINT}:/usr/local/bin/custom_entrypoint.sh
+    entrypoint: ["/usr/local/bin/custom_entrypoint.sh"]
+    command: ["postgres"]
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    restart: no
     env_file:
       - .env
     environment:
-      - POSTGRES_USER
-      - POSTGRES_PASSWORD
-      - POSTGRES_DB
-      - TIMESCALEDB_TELEMETRY=off
+      ENV: ${ENV}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+      PG_DATA: ${PG_DATA}
+      PGUSER: ${PGUSER}
+    labels:
+      ofelia.enabled: "true"
+      # Schedule job to backup timescaledb, commands can be changed to instead run scripts in order to log more data
+      # Perform full backup every day at 00:00
+      ofelia.job-exec.full-backup.user: "postgres"
+      ofelia.job-exec.full-backup.schedule: "0 0 * * *"
+      ofelia.job-exec.full-backup.command: "pgbackrest --stanza=db --type=full --log-level-stderr=info backup"
+      # Perform diff backup every 15 minutes (900 seconds)
+      ofelia.job-exec.diff-backup.schedule: "@every 900s"
+      ofelia.job-exec.diff-backup.user: "postgres"
+      ofelia.job-exec.diff-backup.command: "pgbackrest --stanza=db --type=diff --log-level-stderr=info backup"
+
+
+  # Service for running shedule job to backup timescaledb
+  # https://github.com/mcuadros/ofelia
+  ofelia:
+    image: ${OFELIA_IMAGE}
+    depends_on:
+      timescaledb:
+        condition: service_healthy
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ${OFELIA_LOGS}:/tmp/logs:rw
+    command: daemon --docker
+    env_file:
+      - .env
+    labels:
+      # Save logs locally and via email reports
+      ofelia.save-folder: "./tmp/logs"
+      ofelia.smtp-host: "${OFELIA_SMTP_HOST}"
+      ofelia.smtp-port: "${OFELIA_SMTP_PORT}"
+      ofelia.smtp-user: "${OFELIA_SMTP_USER}"
+      ofelia.smtp-password: "${OFELIA_SMTP_PASSWORD}"
+      ofelia.email-to: "${OFELIA_EMAIL_TO}"
+      ofelia.email-from: "${OFELIA_EMAIL_FROM}"
+
+
+

infra/scripts/clean_start.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+
+# Step [1]: For full clean start, remove the below directories
+# Step [2]: Run the script
+# Step [3]: Run the binary located in ./database/src/bin/tables_migration.rs
+
+# Source the .env
+# Assuming env_loader.sh is in the same directory as this script
+source "$(dirname "$0")/env_loader.sh"
+read_env
+
+# Just in case stop the docker-compose
+CONTAINER_ID=$(docker ps --filter "ancestor=$TIMESCALEDB_IMAGE" --format "{{.ID}}")
+
+# Define maximum wait time in seconds (20 seconds)
+MAX_WAIT=20
+# Define sleep SLEEP_INTERVAL in seconds
+SLEEP_INTERVAL=2
+
+if [[ -n "$CONTAINER_ID" ]]; then
+  echo "Container found for $TIMESCALEDB_IMAGE, initiating shutdown..."
+
+  # Attempt to gracefully stop the container
+  sudo docker compose down
+
+  WAIT_TIME=0
+
+  # Wait for the container to stop
+  while docker ps --filter "id=$CONTAINER_ID" --format "{{.ID}}" | grep -q "$CONTAINER_ID"; do
+    echo "Waiting for container $CONTAINER_ID to stop... ($WAIT_TIME seconds)"
+    sleep $SLEEP_INTERVAL
+    ((WAIT_TIME += SLEEP_INTERVAL))
+    if ((WAIT_TIME >= MAX_WAIT)); then
+      echo "Maximum wait time reached. Proceeding with the next steps."
+      break
+    fi
+  done
+
+  if ((WAIT_TIME < MAX_WAIT)); then
+    echo "Container $CONTAINER_ID has been successfully stopped."
+  fi
+else
+  echo "No running container found for $TIMESCALEDB_IMAGE."
+fi
+
+# Define the base directory as the path to the infra directory
+BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)"
+
+directories=(
+  "${BASE_DIR}/${TIMESCALEDB_DATA}"
+  "${BASE_DIR}/${TIMESCALEDB_BACKUPS}"
+  "${BASE_DIR}/${TIMESCALEDB_LOGS}"
+  "${BASE_DIR}/${TIMESCALEDB_PGBACKREST_CONFIG}"
+  "${BASE_DIR}/${OFELIA_LOGS}"
+)
+
+printf "\n------------- Tyding up the directories -------------\n"
+for dir in "${directories[@]}"; do
+  if [ ! -d "$dir" ]; then
+    echo "Creating directory $dir"
+    mkdir -p "$dir"
+  fi
+
+  sudo chown -R $USER:$USER "$dir"
+  chmod -R 777 "$dir"
+done
+
+# Configuration file path
+pgbackrest_conf="$BASE_DIR/config/pgbackrest.conf"
+
+# Call the setup script with the configuration file path
+echo "Generating pgbackrest_config..."
+"$BASE_DIR/scripts/generate_pgbackrest_config.sh" "$pgbackrest_conf"
+
+# Change to the infra directory where the docker-compose.yaml file is located
+cd "$BASE_DIR"
+
+# Define PostgreSQL data directory
+POSTGRESQL_DATA_DIR="${PG_DATA}/data"  # Source this path from .env
+# PostgreSQL configuration file
+POSTGRESQL_CONF="$POSTGRESQL_DATA_DIR/postgresql.conf"
+BACKUP_MARKER="db"
+
+# Change to the infra directory where the docker-compose.yaml file is located
+cd "$BASE_DIR"
+
+# Start Docker Compose in detached mode
+echo "Starting Docker Compose..."
+sudo docker compose up -d --no-deps --force-recreate --remove-orphans
+
+# Function to check database readiness
+wait_for_db_ready() {
+  echo "Waiting for TimescaleDB to be ready..."
+  local WAIT_TIME=0
+  local CONTAINER_ID=""
+
+  while ((WAIT_TIME < MAX_WAIT)); do
+    CONTAINER_ID=$(docker ps --filter "ancestor=$TIMESCALEDB_IMAGE" --format "{{.ID}}")
+    if [[ -z "$CONTAINER_ID" ]]; then
+      echo "No container found for $TIMESCALEDB_IMAGE, retrying..."
+      sleep $SLEEP_INTERVAL
+      ((WAIT_TIME += SLEEP_INTERVAL))
+      continue
+    fi
+
+    # Check the Docker logs for the readiness message
+    if sudo docker logs $CONTAINER_ID 2>&1 | grep -q "database system is ready to accept connections"; then
+      echo "TimescaleDB is now ready."
+      return 0
+    fi
+
+    echo "Waiting for TimescaleDB to be ready... ${WAIT_TIME}s elapsed"
+    sleep $SLEEP_INTERVAL
+    ((WAIT_TIME += SLEEP_INTERVAL))
+  done
+
+  echo "Timeout waiting for TimescaleDB to be ready after ${MAX_WAIT}s."
+  return 1
+}
+
+# Wait for TimescaleDB to be ready
+if wait_for_db_ready; then
+  printf "\n------------- Updating PostgreSQL configuration -------------\n"
+
+  CONTAINER_ID=$(docker ps --filter "ancestor=$TIMESCALEDB_IMAGE" --format "{{.ID}}")
+  if [[ -z "$CONTAINER_ID" ]]; then
+    echo "Failed to find a running container for $TIMESCALEDB_IMAGE. Exiting."
+    exit 1
+  fi
+
+  sudo docker exec -u root $CONTAINER_ID bash -c "
+    echo -e '\n# Custom PostgreSQL Configurations' >> $POSTGRESQL_CONF
+    echo 'archive_mode = on' >> $POSTGRESQL_CONF
+    echo 'archive_command = '\"'pgbackrest --stanza=$BACKUP_MARKER archive-push %p'\"'' >> $POSTGRESQL_CONF
+    echo 'max_wal_senders = 3' >> $POSTGRESQL_CONF
+    echo 'wal_level = logical' >> $POSTGRESQL_CONF
+  "
+  echo "PostgreSQL configuration updated."
+
+  # Restart docker to apply the changes
+  echo "Restarting the container..."
+  sudo docker restart $CONTAINER_ID
+else
+  echo "Failed to confirm TimescaleDB readiness. Check logs for more details."
+fi
+
+# Wait for TimescaleDB to be ready after the restart
+if wait_for_db_ready; then
+  printf "\n------------- Proceeding with pgBackRest setup -------------\n"
+  CONTAINER_ID=$(docker ps --filter "ancestor=$TIMESCALEDB_IMAGE" --format "{{.ID}}")
+  if [[ -z "$CONTAINER_ID" ]]; then
+    echo "Failed to find a running container for $TIMESCALEDB_IMAGE after restart. Exiting."
+    exit 1
+  fi
+
+  sleep 2
+  # Execute pgBackRest stanza-create
+  sudo docker exec -u root $CONTAINER_ID bash -c "
+    pgbackrest --stanza=$BACKUP_MARKER --log-level-console=info --pg1-path=/home/postgres/pgdata/data --repo1-path=/var/lib/pgbackrest stanza-create
+  "
+  echo "pgBackRest stanza-create executed."
+
+  # Fix permissions for the pgBackRest backup directory
+  sudo docker exec -u root $CONTAINER_ID bash -c "
+    chown -R postgres:postgres /var/lib/pgbackrest
+    chmod -R 700 /var/lib/pgbackrest
+    mkdir -p /var/log/pgbackrest
+    chown -R postgres:postgres /var/log/pgbackrest
+    chmod -R 770 /var/log/pgbackrest
+    chown -R postgres:postgres /tmp/pgbackrest
+    chmod -R 770 /tmp/pgbackrest
+    mkdir -p /var/log/pgbackrest
+    chown -R postgres:postgres /var/lib/pgbackrest
+    chmod -R 770 /var/lib/pgbackrest
+  "
+  echo "Permissions fixed for pgBackRest backup and log directories."
+
+  # Execute pgBackRest check
+  sudo docker exec -u postgres $CONTAINER_ID bash -c "
+    pgbackrest --stanza=$BACKUP_MARKER --log-level-console=info check
+  "
+  echo "pgBackRest check executed successfully."
+else
+  echo "Failed to confirm TimescaleDB readiness after restart. Check logs for more details."
+  exit 1
+fi

infra/scripts/custom_entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+echo "Starting the original entrypoint script..."
+
+docker-entrypoint.sh "$@"
+echo "Original entrypoint script has been called."
+
+# Keep the container from exiting
+tail -f /dev/null

infra/scripts/env_loader.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+read_env() {
+  local filePath="${1:-.env}"
+
+  if [ ! -f "$filePath" ]; then
+    echo "Missing ${filePath}"
+    exit 1
+  fi
+
+  echo "Reading $filePath"
+  while IFS='=' read -r key value; do
+    key=$(echo "$key" | awk '{$1=$1};1' | tr -d '\r')
+    value=$(echo "$value" | sed 's/#.*//' | awk '{$1=$1};1' | tr -d '\r')
+
+    if [[ -n $key ]] && [[ -n $value ]]; then
+      if [[ "$value" =~ ^\".*\"$ || "$value" =~ ^\'.*\'$ ]]; then
+        value="${value:1:-1}"  # Strip the quotes
+      fi
+      export "$key"="$value"
+    fi
+  done < "$filePath"
+}

infra/scripts/generate_pgbackrest_config.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# The first argument is the path where the pgbackrest.conf should be created/overwritten
+config_path="$1"
+
+# Check if the configuration path was provided
+if [ -z "$config_path" ]; then
+    echo "Usage: $0 <path_to_pgbackrest.conf>"
+    exit 1
+fi
+
+# Source the .env
+# Assuming env_loader.sh is in the same directory as this script
+source "$(dirname "$0")/env_loader.sh"
+read_env
+
+# Creating the pgbackrest configuration file
+echo "Creating pgbackrest configuration at $config_path"
+
+# Use 'cat' to write the contents to the configuration file
+cat >"$config_path" <<EOF
+[db]
+pg1-path=${PG_DATA}/data
+pg1-socket-path=/var/run/postgresql
+[global]
+repo1-retention-full=3
+repo1-retention-diff=4
+repo1-path=/var/lib/pgbackrest
+[global:archive-push]
+compress-level=3 
+EOF
+
+echo "pgbackrest configuration has been created."