From 4793003946e27a93c437713e0a99e48ded63e1f7 Mon Sep 17 00:00:00 2001 From: Nana-EC <56320167+Nana-EC@users.noreply.github.com> Date: Tue, 22 Dec 2020 17:28:44 -0600 Subject: [PATCH] Updated for super and owner users Signed-off-by: Nana-EC <56320167+Nana-EC@users.noreply.github.com> --- docker-compose.override.yml | 4 ++ docker-compose.yml | 14 +++++ docs/operations.md | 48 +++++++++++++---- .../src/main/resources/db/scripts/init_v2.sql | 53 ++++++++++++------- .../time-scale-migration/csvBackupTables.sql | 2 - .../time-scale-migration/csvRestoreTables.sql | 2 - .../time-scale-migration/migration.config | 2 +- .../time-scale-migration/timeScaleDbInit.sql | 45 ---------------- .../timeScaleDbMigration.sh | 2 +- 9 files changed, 93 insertions(+), 79 deletions(-) create mode 100644 docker-compose.override.yml delete mode 100644 hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbInit.sql diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 00000000000..f263f9124f1 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,4 @@ +version: "3.3" +services: + tsdb: + entrypoint: ["echo", "TimeScaleDB service is disabled"] diff --git a/docker-compose.yml b/docker-compose.yml index e7eddbd1679..cbd989a4c7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,3 +67,17 @@ services: tty: true ports: - 5551:5551 + + tsdb: + image: timescaledev/timescaledb-ha:pg12-ts2.0.0-rc3 + restart: unless-stopped + stop_grace_period: 2m + stop_signal: SIGTERM + tty: true + environment: + POSTGRES_PASSWORD: mirror_node_pass + volumes: + - ./tsdb:/var/lib/postgresql/data + - ./hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql/:/docker-entrypoint-initdb.d/init_v2.sql + ports: + - 6432:5432 diff --git a/docs/operations.md b/docs/operations.md index a61cab08fa9..47b6bdd42ad 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -95,33 +95,61 @@ sudo journalctl -fu hedera-mirror-importer.service ### v1 to v2 Data Migration -To support time series logic the Mirror Node db schema shifted from PostgeSQL (v1) to TimeScaleDB (v2) +To support time series logic the Mirror Node DB schema shifted from PostgeSQL (v1) to TimeScaleDB (v2). +Adopting the recommended steps [Migrating from a Different PostgreSQL Database](https://docs.timescale.com/latest/getting-started/migrating-data#different-db) we + + For mirror node operators running v1 db schema looking to upgrade to v2 the following steps can be taken 1. Setup a new database using TimeScale -Docker installation recommended - https://docs.timescale.com/latest/getting-started/installation/docker/installation-docker + Docker installation steps are recommended - https://docs.timescale.com/latest/getting-started/installation/docker/installation-docker + + To install using docker-compose: + Update the `docker-compose.override.yml` file to disable postgres instead of TimeScaleDB + ```yaml + version: "3.3" + services: + db: + entrypoint: ["echo", "PostgreSQL db is disabled"] + ``` + + Start up a TimescaleDB service: + ```shell script + $ docker-compose up tsdb + ``` + + Note: If the new db is running on the same server node as the original db, then the port must be updated to something other than 5432. + The `tsdb` port can be updated to a different port e.g. 6432 as follows: + ```yaml + ... + services: + ... + tsdb: + ports: + - 6432:5432 + ``` 2. Create DB & Init Schema -The init script for v2 at `hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql` may be used to create teh db, users, schema, extensions and ensure all permissions are set. -This may be run manually against the db node or in teh docker case mounted under `/docker-entrypoint-initdb.d/` on the docker container -> **_NOTE:_** The following steps assume the database, users and schema have been created + The init script for v2 at `hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql` may be used to create the db, users, schema, extensions and ensure all permissions are set. + This may be run manually against the db node. In the docker-compose case this file is already mounted under `/docker-entrypoint-initdb.d/` on the docker container and run on startup. + + > **_NOTE:_** The following steps assume the database, users and schema have been created 3. Configure migration properties -A properties file contains variable for easy running. These options include variables such as db names, passwords, users, hosts for both the existing db and the new db. -Updated the values at `hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config` appropriately + A properties file contains db variable for easy running. These options include variables such as db names, passwords, users, hosts for both the existing db and the new db. + + Update the values at `hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config` appropriately for your db setup. 4. Run migration script -From the `hedera-mirror-importer/src/main/resources/db` directory run + From the `hedera-mirror-importer/src/main/resources/db` directory run ```shell script $ ./scripts/time-scale-migration/timeScaleDbMigration.sh ``` -Adopting the recommended steps [Migrating from a Different PostgreSQL Database](https://docs.timescale.com/latest/getting-started/migrating-data#different-db) we - ## Monitor The monitor is a Java-based application and should be able to run on any platform that Java supports. That said, we diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql b/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql index 68c29decb3e..373a0288f42 100644 --- a/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql +++ b/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql @@ -1,46 +1,63 @@ -- init the timescale db mirror node db -- Change the values below if you are not installing via Docker +\set db_host 'localhost' +\set db_port 6432 \set db_name 'mirror_node' -\set importer_user 'mirror_node' -\set importer_password 'mirror_node_pass' -\set importer_user 'mirror_node' +\set db_super_user 'postgres' +\set db_owner 'mirror_node' +\set owner_password 'mirror_node_pass' +\set importer_user 'mirror_importer' +\set importer_password 'mirror_importer_pass' \set grpc_user 'mirror_grpc' \set grpc_password 'mirror_grpc_pass' \set rest_user 'mirror_api' \set rest_password 'mirror_api_pass' -\set schema_name 'mirror_node' +\set schema_name 'mirrornode' + +-- create owner user +create user :db_owner with login createrole password :'owner_password'; -- create primary user and db -create database :db_name; -\c :db_name -create extension if not exists timescaledb cascade; +create database :db_name with owner :db_owner; +\c :db_name :db_owner -- create users -create user :importer_user with login createrole password :'importer_password'; +create user :importer_user with login password :'importer_password'; create role viewer; create user :grpc_user with login password :'grpc_password' in role viewer; create user :rest_user with login password :'rest_password' in role viewer; -- grant connect access to api users -grant connect on database :db_name to :grpc_user; -grant connect on database :db_name to :rest_user; +grant connect on database :db_name to viewer; -- schema -create schema if not exists :schema_name; +create schema if not exists :schema_name authorization :db_owner; grant usage on schema :schema_name to public; -grant all privileges on all tables in schema :schema_name to :importer_user; -- alter search path for given schema +alter user :db_owner set search_path = :schema_name, public; alter user :importer_user set search_path = :schema_name, public; alter user :grpc_user set search_path = :schema_name, public; alter user :rest_user set search_path = :schema_name, public; --- grant select privileges on past and future tables to api users +-- grant select privileges on past and future tables and sequences to users +grant all privileges on all tables in schema :schema_name to :db_owner; +grant all privileges on all sequences in schema :schema_name to :db_owner; grant select on all tables in schema :schema_name to :importer_user; grant select on all tables in schema :schema_name to viewer; -alter default privileges for role :importer_user in schema :schema_name grant select on tables to viewer; - --- add extensions -create extension if not exists timescaledb cascade; -create extension pg_stat_statements; +grant select on all sequences in schema :schema_name to :importer_user; +grant select on all sequences in schema :schema_name to viewer; +alter default privileges in schema :schema_name grant select on tables to :importer_user; +alter default privileges in schema :schema_name grant select on tables to viewer; +alter default privileges in schema :schema_name grant select on sequences to :importer_user; +alter default privileges in schema :schema_name grant select on sequences to viewer; + +-- add extensions, ensuring they're available to new schema +-- drop extension if exists timescaledb; +\c :db_name :db_super_user +drop extension if exists timescaledb; +-- must reconnect otherwise fails with "Start a new session and execute CREATE EXTENSION as the first command. Make sure to pass the "-X" flag to psql." +\c :db_name :db_super_user +create extension if not exists timescaledb schema :schema_name cascade; +create extension if not exists pg_stat_statements schema :schema_name; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvBackupTables.sql b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvBackupTables.sql index 82eddd8d0f6..deab20c3f1f 100644 --- a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvBackupTables.sql +++ b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvBackupTables.sql @@ -18,8 +18,6 @@ \copy (select * from file_data) to file_data.csv delimiter ',' csv; -\copy (select * from flyway_schema_history) to flyway_schema_history.csv delimiter ',' csv; - \copy (select * from live_hash) to live_hash.csv delimiter ',' csv; \copy (select * from non_fee_transfer) to non_fee_transfer.csv delimiter ',' csv; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvRestoreTables.sql b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvRestoreTables.sql index b56f123ae14..dc77e28c6c9 100644 --- a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvRestoreTables.sql +++ b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/csvRestoreTables.sql @@ -18,8 +18,6 @@ \copy file_data from file_data.csv csv; -\copy flyway_schema_history from flyway_schema_history.csv csv; - \copy live_hash from live_hash.csv csv; \copy non_fee_transfer from non_fee_transfer.csv csv; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config index 406907cadfc..61e459c6228 100644 --- a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config +++ b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/migration.config @@ -1,6 +1,6 @@ OLD_DB_HOST=localhost OLD_DB_NAME=mirror_node -OLD_DB_PORT=7432 +OLD_DB_PORT=5432 OLD_DB_USER=mirror_node OLD_PASSWORD=mirror_node_pass NEW_DB_HOST=localhost diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbInit.sql b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbInit.sql deleted file mode 100644 index a25f438fa49..00000000000 --- a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbInit.sql +++ /dev/null @@ -1,45 +0,0 @@ --- init the timescale db mirror node db --- Change the values below if you are not installing via Docker - -\set db_name 'mirror_node' -\set db_user 'mirror_node' -\set db_password 'mirror_node_pass' -\set db_owner 'mirror_node' -\set grpc_user 'mirror_grpc' -\set grpc_password 'mirror_grpc_pass' -\set rest_user 'mirror_api' -\set rest_password 'mirror_api_pass' -\set schema_name 'mirror_node' - --- create primary user and db -create user :db_user with login createrole password :'db_password'; -create database :db_name with owner :db_owner; - --- create api users -create user :grpc_user with login password :'grpc_password'; -create user :rest_user with login password :'rest_password'; - --- is it necessary to explicitly grant the following? -grant all privileges on database :db_name to :db_user; - --- grant connect access to api users -grant connect on database :db_name to :grpc_user; -grant connect on database :db_name to :rest_user; - -\c :db_name - --- schema -create schema if not exists :schema_name; -grant usage on schema :schema_name to public; -grant all privileges on all tables in schema :schema_name to :db_user; - --- alter search path for given schema -alter user :db_user set search_path = :schema_name, public; -alter user :grpc_user set search_path = :schema_name, public; -alter user :rest_user set search_path = :schema_name, public; - --- grant select privileges on past and future tables to api users -alter default privileges in schema :schema_name grant select on tables to :grpc_user; -alter default privileges in schema :schema_name grant select on tables to :rest_user; -grant select on all tables in schema :schema_name to :grpc_user; -grant select on all tables in schema :schema_name to :rest_user; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbMigration.sh b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbMigration.sh index 77bb2580663..c97d1c33509 100755 --- a/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbMigration.sh +++ b/hedera-mirror-importer/src/main/resources/db/scripts/time-scale-migration/timeScaleDbMigration.sh @@ -59,7 +59,7 @@ start_time="$(date -u +%s)" echo "Migrating Mirror Node Data from Postgres($OLD_DB_HOST:$OLD_DB_PORT) to TimeScaleDb($NEW_DB_HOST:$NEW_DB_PORT)..." echo "1. Backing up flyway table schema from Postgres($OLD_DB_HOST:$OLD_DB_PORT)..." -PGPASSWORD=${OLD_PASSWORD} pg_dump -h $OLD_DB_HOST -p $OLD_DB_PORT -U $OLD_DB_USER --section=pre-data --table public.flyway_schema_history -f mirror_node_${start_time}.bak mirror_node +PGPASSWORD=${OLD_PASSWORD} pg_dump -h $OLD_DB_HOST -p $OLD_DB_PORT -U $OLD_DB_USER --table public.flyway_schema_history -f mirror_node_${start_time}.bak mirror_node echo "2. Restoring flyway_schema_history to TimeScaleDb($NEW_DB_HOST:$NEW_DB_PORT)..." PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER