Updated for super and owner users

Signed-off-by: Nana-EC <[email protected]>

docker-compose.override.yml

@@ -0,0 +1,4 @@
+version: "3.3"
+services:
+  tsdb:
+    entrypoint: ["echo", "TimeScaleDB service is disabled"]

docker-compose.yml

@@ -67,3 +67,17 @@ services:
     tty: true
     ports:
       - 5551:5551
+
+  tsdb:
+    image: timescaledev/timescaledb-ha:pg12-ts2.0.0-rc3
+    restart: unless-stopped
+    stop_grace_period: 2m
+    stop_signal: SIGTERM
+    tty: true
+    environment:
+      POSTGRES_PASSWORD: mirror_node_pass
+    volumes:
+      - ./tsdb:/var/lib/postgresql/data
+      - ./hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql/:/docker-entrypoint-initdb.d/init_v2.sql
+    ports:
+      - 6432:5432

docs/operations.md

@@ -95,33 +95,61 @@ sudo journalctl -fu hedera-mirror-importer.service
 
 ### v1 to v2 Data Migration
 
-To support time series logic the Mirror Node db schema shifted from PostgeSQL (v1) to TimeScaleDB (v2)
+To support time series logic the Mirror Node DB schema shifted from PostgeSQL (v1) to TimeScaleDB (v2).
+Adopting the recommended steps [Migrating from a Different PostgreSQL Database](https://docs.timescale.com/latest/getting-started/migrating-data#different-db) we
+
+
 For mirror node operators running v1 db schema looking to upgrade to v2 the following steps can be taken
 
 1. Setup a new database using TimeScale
-Docker installation recommended - https://docs.timescale.com/latest/getting-started/installation/docker/installation-docker
+    Docker installation steps are recommended - https://docs.timescale.com/latest/getting-started/installation/docker/installation-docker
+
+    To install using docker-compose:
+    Update the  `docker-compose.override.yml` file to disable postgres instead of TimeScaleDB
 
+    ```yaml
+    version: "3.3"
+    services:
+      db:
+        entrypoint: ["echo", "PostgreSQL db is disabled"]
+    ```
+
+    Start up a TimescaleDB service:
+    ```shell script
+    $ docker-compose up tsdb
+    ```
+
+    Note: If the new db is running on the same server node as the original db, then the port must be updated to something other than 5432.
+    The `tsdb` port can be updated to a different port e.g. 6432 as follows:
+    ```yaml
+    ...
+    services:
+      ...
+      tsdb:
+        ports:
+          - 6432:5432
+    ```
 
 2. Create DB & Init Schema
-The init script for v2 at `hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql` may be used to create teh db, users, schema, extensions and ensure all permissions are set.
-This may be run manually against the db node or in teh docker case mounted under `/docker-entrypoint-initdb.d/` on the docker container
 
-> **_NOTE:_** The following steps assume the database, users and schema have been created
+    The init script for v2 at `hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql` may be used to create the db, users, schema, extensions and ensure all permissions are set.
+    This may be run manually against the db node. In the docker-compose case this file is already mounted under `/docker-entrypoint-initdb.d/` on the docker container and run on startup.
+
+    > **_NOTE:_** The following steps assume the database, users and schema have been created
 
 3. Configure migration properties
-A properties file contains variable for easy running. These options include variables such as db names, passwords, users, hosts for both the existing db and the new db.
 
-Updated the values at `hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config` appropriately
+    A properties file contains db variable for easy running. These options include variables such as db names, passwords, users, hosts for both the existing db and the new db.
+
+    Update the values at `hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config` appropriately for your db setup.
 
 4. Run migration script
 
-From the `hedera-mirror-importer/src/main/resources/db` directory run
+    From the `hedera-mirror-importer/src/main/resources/db` directory run
     ```shell script
     $ ./scripts/time-scale-migration/timeScaleDbMigration.sh
     ```
 
-Adopting the recommended steps [Migrating from a Different PostgreSQL Database](https://docs.timescale.com/latest/getting-started/migrating-data#different-db) we
-
 ## Monitor
 
 The monitor is a Java-based application and should be able to run on any platform that Java supports. That said, we

hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql

@@ -1,46 +1,63 @@
 -- init the timescale db mirror node db
 -- Change the values below if you are not installing via Docker
 
+\set db_host 'localhost'
+\set db_port 6432
 \set db_name 'mirror_node'
-\set importer_user 'mirror_node'
-\set importer_password 'mirror_node_pass'
-\set importer_user 'mirror_node'
+\set db_super_user 'postgres'
+\set db_owner 'mirror_node'
+\set owner_password 'mirror_node_pass'
+\set importer_user 'mirror_importer'
+\set importer_password 'mirror_importer_pass'
 \set grpc_user 'mirror_grpc'
 \set grpc_password 'mirror_grpc_pass'
 \set rest_user 'mirror_api'
 \set rest_password 'mirror_api_pass'
-\set schema_name 'mirror_node'
+\set schema_name 'mirrornode'
+
+-- create owner user
+create user :db_owner with login createrole password :'owner_password';
 
 -- create primary user and db
-create database :db_name;
-\c :db_name
-create extension if not exists timescaledb cascade;
+create database :db_name with owner :db_owner;
+\c :db_name :db_owner
 
 -- create users
-create user :importer_user with login createrole password :'importer_password';
+create user :importer_user with login password :'importer_password';
 create role viewer;
 create user :grpc_user with login password :'grpc_password' in role viewer;
 create user :rest_user with login password :'rest_password' in role viewer;
 
 -- grant connect access to api users
-grant connect on database :db_name to :grpc_user;
-grant connect on database :db_name to :rest_user;
+grant connect on database :db_name to viewer;
 
 -- schema
-create schema if not exists :schema_name;
+create schema if not exists :schema_name authorization :db_owner;
 grant usage on schema :schema_name to public;
-grant all privileges on all tables in schema :schema_name to :importer_user;
 
 -- alter search path for given schema
+alter user :db_owner set search_path = :schema_name, public;
 alter user :importer_user set search_path = :schema_name, public;
 alter user :grpc_user set search_path = :schema_name, public;
 alter user :rest_user set search_path = :schema_name, public;
 
--- grant select privileges on past and future tables to api users
+-- grant select privileges on past and future tables and sequences to users
+grant all privileges on all tables in schema :schema_name to :db_owner;
+grant all privileges on all sequences in schema :schema_name to :db_owner;
 grant select on all tables in schema :schema_name to :importer_user;
 grant select on all tables in schema :schema_name to viewer;
-alter default privileges for role :importer_user in schema :schema_name grant select on tables to viewer;
-
--- add extensions
-create extension if not exists timescaledb cascade;
-create extension pg_stat_statements;
+grant select on all sequences in schema :schema_name to :importer_user;
+grant select on all sequences in schema :schema_name to viewer;
+alter default privileges in schema :schema_name grant select on tables to :importer_user;
+alter default privileges in schema :schema_name grant select on tables to viewer;
+alter default privileges in schema :schema_name grant select on sequences to :importer_user;
+alter default privileges in schema :schema_name grant select on sequences to viewer;
+
+-- add extensions, ensuring they're available to new schema
+-- drop extension if exists timescaledb;
+\c :db_name :db_super_user
+drop extension if exists timescaledb;
+-- must reconnect otherwise fails with "Start a new session and execute CREATE EXTENSION as the first command. Make sure to pass the "-X" flag to psql."
+\c :db_name :db_super_user
+create extension if not exists timescaledb schema :schema_name cascade;
+create extension if not exists pg_stat_statements schema :schema_name;

hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvBackupTables.sql

@@ -18,8 +18,6 @@
 
 \copy (select * from file_data) to file_data.csv delimiter ',' csv;
 
-\copy (select * from flyway_schema_history) to flyway_schema_history.csv delimiter ',' csv;
-
 \copy (select * from live_hash) to live_hash.csv delimiter ',' csv;
 
 \copy (select * from non_fee_transfer) to non_fee_transfer.csv delimiter ',' csv;

hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvRestoreTables.sql

@@ -18,8 +18,6 @@
 
 \copy file_data from file_data.csv csv;
 
-\copy flyway_schema_history from flyway_schema_history.csv csv;
-
 \copy live_hash from live_hash.csv csv;
 
 \copy non_fee_transfer from non_fee_transfer.csv csv;

hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config

@@ -1,6 +1,6 @@
 OLD_DB_HOST=localhost
 OLD_DB_NAME=mirror_node
-OLD_DB_PORT=7432
+OLD_DB_PORT=5432
 OLD_DB_USER=mirror_node
 OLD_PASSWORD=mirror_node_pass
 NEW_DB_HOST=localhost

hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbMigration.sh

@@ -59,7 +59,7 @@ start_time="$(date -u +%s)"
 echo "Migrating Mirror Node Data from Postgres($OLD_DB_HOST:$OLD_DB_PORT) to TimeScaleDb($NEW_DB_HOST:$NEW_DB_PORT)..."
 
 echo "1. Backing up flyway table schema from Postgres($OLD_DB_HOST:$OLD_DB_PORT)..."
-PGPASSWORD=${OLD_PASSWORD} pg_dump -h $OLD_DB_HOST -p $OLD_DB_PORT -U $OLD_DB_USER --section=pre-data --table public.flyway_schema_history -f mirror_node_${start_time}.bak mirror_node
+PGPASSWORD=${OLD_PASSWORD} pg_dump -h $OLD_DB_HOST -p $OLD_DB_PORT -U $OLD_DB_USER --table public.flyway_schema_history -f mirror_node_${start_time}.bak mirror_node
 
 echo "2. Restoring flyway_schema_history to TimeScaleDb($NEW_DB_HOST:$NEW_DB_PORT)..."
 PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER <mirror_node_${start_time}.bak