diff --git a/.github/ct-install.yaml b/.github/ct-install.yaml index c3d6bf62b77..f50b3487b64 100644 --- a/.github/ct-install.yaml +++ b/.github/ct-install.yaml @@ -3,6 +3,7 @@ chart-repos: - fluxcd=https://charts.fluxcd.io - loki=https://grafana.github.io/helm-charts - prometheus=https://prometheus-community.github.io/helm-charts + - timescaledb=https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/ - traefik=https://helm.traefik.io/traefik check-version-increment: false charts: diff --git a/.github/ct-lint.yaml b/.github/ct-lint.yaml index c1f51a37e32..5034ae2b4a5 100644 --- a/.github/ct-lint.yaml +++ b/.github/ct-lint.yaml @@ -3,5 +3,6 @@ chart-repos: - fluxcd=https://charts.fluxcd.io - loki=https://grafana.github.io/helm-charts - prometheus=https://prometheus-community.github.io/helm-charts + - timescaledb=https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/ - traefik=https://helm.traefik.io/traefik check-version-increment: false diff --git a/charts/hedera-mirror/requirements.lock b/charts/hedera-mirror/requirements.lock index b0637cb04e0..49ac6e5c402 100644 --- a/charts/hedera-mirror/requirements.lock +++ b/charts/hedera-mirror/requirements.lock @@ -10,15 +10,15 @@ dependencies: version: 0.12.0-rc1 - name: postgresql-ha repository: https://charts.bitnami.com/bitnami - version: 6.3.2 + version: 6.3.4 - name: redis repository: https://charts.bitnami.com/bitnami - version: 12.2.3 + version: 12.2.4 - name: hedera-mirror-rest repository: file://../hedera-mirror-rest version: 0.12.0-rc1 - name: timescaledb-multinode repository: https://raw.githubusercontent.com/timescale/timescaledb-kubernetes/master/charts/repo/ version: 0.7.0 -digest: sha256:b261bda6d871b9ddc44e06ad21b51429a14028ca8db2ff330d54a11e5109e630 -generated: "2020-12-21T15:48:49.454221-06:00" +digest: sha256:5b4f3a8e19c559e3507fce9339b86f91a82da8cbe57eee076c5d67f085c5b3f8 +generated: "2021-01-04T17:28:47.546372-06:00" diff --git a/charts/hedera-mirror/templates/job-timescaledb-init.yaml b/charts/hedera-mirror/templates/job-timescaledb-init.yaml index 5ab5b38c750..278b23fd87c 100644 --- a/charts/hedera-mirror/templates/job-timescaledb-init.yaml +++ b/charts/hedera-mirror/templates/job-timescaledb-init.yaml @@ -2,20 +2,19 @@ apiVersion: batch/v1 kind: Job metadata: - labels: - {{- include "hedera-mirror.labels" . | nindent 4 }} + labels: {{- include "hedera-mirror.labels" . | nindent 4 }} name: {{ include "hedera-mirror.dbHost" . }}-init-job namespace: {{ include "hedera-mirror.namespace" . }} annotations: "helm.sh/hook": post-install "helm.sh/hook-delete-policy": hook-succeeded spec: - backoffLimit: 4 + backoffLimit: 1 template: spec: containers: - name: init-mirrornode-db - image: timescaledev/timescaledb-ha:pg12-ts2.0.0-rc3 + image: "{{ .Values.timescaledb.image.repository }}:{{ .Values.timescaledb.image.tag }}" command: - sh - -c @@ -23,20 +22,30 @@ spec: set -e while ! pg_isready -U postgres -h {{ include "hedera-mirror.dbHost" . }}-data; do sleep 1; done; - psql --echo-queries -d "${ACCESS_SVC_CONNSTR_POSTGRES}" --set ON_ERROR_STOP=1 -f ${DB_INIT_FILE} + psql --echo-queries -d "${ACCESS_SVC_CONNSTR_POSTGRES}" --set ON_ERROR_STOP=1 -f ${DB_INIT_DIR}/users_v2.sql + echo 'Completed db initialization and user creation for mirror node' + + psql --echo-queries -d "${ACCESS_SVC_CONNSTR_MIRRORNODE}" --set ON_ERROR_STOP=1 -f ${DB_INIT_DIR}/schema_v2.sql + echo 'Completed db schema initialization' + + psql --echo-queries -d "${ACCESS_SVC_CONNSTR_POSTGRES}" --set ON_ERROR_STOP=1 -f ${DB_INIT_DIR}/path_v2.sql + echo 'Completed db search path setting' + + echo 'Completed db initialization for mirror node' env: - name: ACCESS_SVC_CONNSTR_POSTGRES value: host={{ include "hedera-mirror.dbHost" . }} user=postgres connect_timeout=3 sslmode=disable password={{ .Values.timescaledb.credentials.accessNode.superuser }} - - name: DB_INIT_FILE - value: /usr/etc/db-init/init.sql + - name: ACCESS_SVC_CONNSTR_MIRRORNODE + value: host={{ include "hedera-mirror.dbHost" . }} dbname={{ .Values.importer.config.hedera.mirror.importer.db.name }} user={{ .Values.importer.config.hedera.mirror.importer.db.owner }} connect_timeout=3 sslmode=disable password={{ .Values.importer.config.hedera.mirror.importer.db.ownerPassword }} + - name: DB_INIT_DIR + value: /usr/etc/db-init volumeMounts: - - name: timescale-db-init-volume + - name: timescaledb-init-volume mountPath: /usr/etc/db-init volumes: - - name: timescale-db-init-volume + - name: timescaledb-init-volume secret: defaultMode: 420 secretName: {{ include "hedera-mirror.dbHost" . }}-init - restartPolicy: OnFailure - ttlSecondsAfterFinished: 600 - {{- end -}} + restartPolicy: Never +{{- end -}} diff --git a/charts/hedera-mirror/templates/secret-postgresql.yaml b/charts/hedera-mirror/templates/secret-postgresql.yaml index 603bbead508..4297b39e05d 100644 --- a/charts/hedera-mirror/templates/secret-postgresql.yaml +++ b/charts/hedera-mirror/templates/secret-postgresql.yaml @@ -8,7 +8,7 @@ metadata: namespace: {{ include "hedera-mirror.namespace" . }} type: Opaque stringData: - init.sql: |- + init_v1.sql: |- {{- $dbname := .Values.importer.config.hedera.mirror.importer.db.name }} {{- $password := .Values.importer.config.hedera.mirror.importer.db.password }} {{- $username := .Values.importer.config.hedera.mirror.importer.db.username }} diff --git a/charts/hedera-mirror/templates/secret-timescaledb.yaml b/charts/hedera-mirror/templates/secret-timescaledb.yaml index e63a1815aef..719627b0dca 100644 --- a/charts/hedera-mirror/templates/secret-timescaledb.yaml +++ b/charts/hedera-mirror/templates/secret-timescaledb.yaml @@ -4,27 +4,75 @@ kind: Secret metadata: labels: {{- include "hedera-mirror.labels" . | nindent 4 }} - name: {{ printf "%s-timescaledb-init" .Release.Name }} + name: {{ include "hedera-mirror.dbHost" . }}-init namespace: {{ include "hedera-mirror.namespace" . }} type: Opaque stringData: - init.sql: |- + users_v2.sql: |- {{- $dbName := .Values.importer.config.hedera.mirror.importer.db.name }} + {{- $dbOwner := .Values.importer.config.hedera.mirror.importer.db.owner }} + {{- $dbOwnerPassword := .Values.importer.config.hedera.mirror.importer.db.ownerPassword }} {{- $importerUser := .Values.importer.config.hedera.mirror.importer.db.username }} {{- $importerPassword := .Values.importer.config.hedera.mirror.importer.db.password }} {{- $grpcUsername := .Values.grpc.config.hedera.mirror.grpc.db.username }} {{- $grpcPassword := .Values.grpc.config.hedera.mirror.grpc.db.password }} {{- $restUser := .Values.global.rest.username }} {{- $restPassword := .Values.global.rest.password }} - create database {{ $dbName }}; + {{- $dbSchema := .Values.importer.config.hedera.mirror.importer.db.schema }} + + -- create owner user + create user {{ $dbOwner }} with login password '{{ $dbOwnerPassword }}'; + + -- create primary user and db + create database {{ $dbName }} with owner {{ $dbOwner }}; + + -- create roles + create role readonly; + create role readwrite in role readonly; + + -- create users + create user {{ $grpcUsername }} with login password '{{ $grpcPassword }}' in role readonly; + create user {{ $restUser }} with login password '{{ $restPassword }}' in role readonly; + create user {{ $importerUser }} with login password '{{ $importerPassword }}' in role readwrite; + + -- drop timescaledb extension for future install to ensure availability in custom schema + drop extension if exists timescaledb cascade; + schema_v2.sql: |- + -- create schema and set schema user permissions + create schema if not exists {{ $dbSchema }} authorization {{ $dbOwner }}; + grant usage on schema {{ $dbSchema }} to public; + + -- revoke default public permissions on schema + revoke create on schema {{ $dbSchema }} from public; + + -- grant connect and schema access to readonly role + grant connect on database {{ $dbName }} to readonly; + grant usage on schema {{ $dbSchema }} to readonly; + + -- grant select privileges on tables to readonly + -- grant all privileges on all tables in schema {{ $dbSchema }} to {{ $dbOwner }}; + grant select on all tables in schema {{ $dbSchema }} to readonly; + alter default privileges in schema {{ $dbSchema }} grant select on tables to readonly; + + -- grant select privileges on sequences to readonly + -- grant all privileges on all sequences in schema {{ $dbSchema }} to {{ $dbOwner }}; + grant select on all sequences in schema {{ $dbSchema }} to readonly; + alter default privileges in schema {{ $dbSchema }} grant select on sequences to readonly; + + -- grant write privileges on sequences to readwrite + grant insert, update, delete on all tables in schema {{ $dbSchema }} to readwrite; + alter default privileges in schema {{ $dbSchema }} grant insert, update on tables to readwrite; + grant usage on all sequences in schema {{ $dbSchema }} to readwrite; + alter default privileges in schema {{ $dbSchema }} grant usage on sequences to readwrite; + path_v2.sql: |- \c {{ $dbName }}; - create extension if not exists timescaledb cascade; - create user {{ $importerUser }} with login password '{{ $importerPassword }}'; - create role viewer; - create user {{ $grpcUsername }} with login password '{{ $grpcPassword }}' in role viewer; - create user {{ $restUser }} with login password '{{ $restPassword }}' in role viewer; - grant select on all tables in schema public to {{ $importerUser }}; - grant select on all tables in schema public to viewer; - alter default privileges for role {{ $importerUser }} in schema public grant select on tables to viewer; - create extension pg_stat_statements; - {{- end -}} + -- alter search path for given schema + alter user {{ $dbOwner }} set search_path = {{ $dbSchema }}, public; + alter user {{ $importerUser }} set search_path = {{ $dbSchema }}, public; + alter user {{ $grpcUsername }} set search_path = {{ $dbSchema }}, public; + alter user {{ $restUser }} set search_path = {{ $dbSchema }}, public; + + -- add extensions, ensuring they're available to new schema + create extension if not exists timescaledb cascade schema {{ $dbSchema }}; + create extension if not exists pg_stat_statements cascade schema {{ $dbSchema }}; +{{- end -}} diff --git a/charts/hedera-mirror/values.yaml b/charts/hedera-mirror/values.yaml index f75d4ac81c4..1a03dde9306 100644 --- a/charts/hedera-mirror/values.yaml +++ b/charts/hedera-mirror/values.yaml @@ -185,6 +185,9 @@ timescaledb: dataNode: superuser: mirror_node_pass enabled: false + image: + repository: timescaledev/timescaledb-ha + tag: pg12.5-ts2.0.0-p0 persistentVolume: size: 500Gi resources: @@ -200,7 +203,5 @@ timescaledb: parameters: max_wal_size: 8GB # recommended to be 80% of the Volume Size min_wal_size: 2GB # 80% of the WAL Volume Size - shared_buffers: 1GB # recommended to be 25% of available instance memory + shared_buffers: 1GB # recommended to be 25% of available instance memory work_mem: 50MB - - diff --git a/docker-compose.yml b/docker-compose.yml index e7eddbd1679..fe54f4f3bf4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,3 +67,19 @@ services: tty: true ports: - 5551:5551 + + timescaledb: + deploy: + replicas: 0 + image: timescaledev/timescaledb-ha:pg12.5-ts2.0.0-p0 + restart: unless-stopped + stop_grace_period: 2m + stop_signal: SIGTERM + tty: true + environment: + POSTGRES_PASSWORD: mirror_node_pass + volumes: + - ./timescaledb:/var/lib/postgresql/data + - ./hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql:/docker-entrypoint-initdb.d/init_v2.sql + ports: + - 5432:5432 diff --git a/docs/configuration.md b/docs/configuration.md index 09b287bfba6..81fa5def45f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,9 +26,12 @@ value, it is recommended to only populate overridden properties in the custom `a | `hedera.mirror.importer.db.host` | 127.0.0.1 | The IP or hostname used to connect to the database | | `hedera.mirror.importer.db.loadBalance` | true | Whether to enable pgpool load balancing. If false, it sends all reads to the primary db backend instead of load balancing them across the primary and replicas. | | `hedera.mirror.importer.db.name` | mirror_node | The name of the database | -| `hedera.mirror.importer.db.password` | mirror_node_pass | The database password the processor uses to connect. | +| `hedera.mirror.importer.db.owner` | mirror_node | The username of the db user with owner permissions to create and modify the schema | +| `hedera.mirror.importer.db.ownerPassword` | mirror_node_pass | The password for the owner user the processor uses to connect. | +| `hedera.mirror.importer.db.password` | mirror_node_pass | The database password for the Importer user the processor uses to connect. | | `hedera.mirror.importer.db.port` | 5432 | The port used to connect to the database | -| `hedera.mirror.importer.db.username` | mirror_node | The username the processor uses to connect to the database | +| `hedera.mirror.importer.db.schema` | public | The name of the custom schema database objects will be created in. This is applicable from v2 of the data schema | +| `hedera.mirror.importer.db.username` | mirror_node | The Importer username the processor uses to connect to the database | | `hedera.mirror.importer.downloader.accessKey` | "" | The cloud storage access key | | `hedera.mirror.importer.downloader.allowAnonymousAccess` | | Whether the cloud storage bucket allows for anonymous access. | | `hedera.mirror.importer.downloader.balance.batchSize` | 15 | The number of signature files to download per node before downloading the signed files | diff --git a/docs/installation.md b/docs/installation.md index 8b1363cbf55..fa20eb2c4f5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -19,16 +19,31 @@ runnable Mirror Node JAR file in the `target` directory. ## Running Locally ### Database Setup +In addition to OpenJDK 11, you will need to install a database and initialize it. +The Mirror Node utilizes [PostgreSQL](https://postgresql.org) v9.6 or [TimescaleDB](https://docs.timescale.com/latest/main) v2 depending on the version of its database schema. -In addition to OpenJDK 11, you will need to install [PostgreSQL](https://postgresql.org) 9.6 and initialize it. The only -setup required is to create the initial database and owner since [Flyway](https://flywaydb.org) manages the database -schema. The SQL script located at `hedera-mirror-importer/src/main/resources/db/scripts/init.sql` can be used to -accomplish this. Edit the file and change the `db_name`, `db_user`, `db_password` `db_owner`, `grpc_user`, or -`grpc_password` as appropriate. Make sure the application [configuration](configuration.md) matches the values in the -script. Run the script as a DB admin user and check the output carefully to ensure no errors occurred. +For both databases, since [Flyway](https://flywaydb.org) will manage the database schema, the only required setup steps include: +* creating the database, users, schema, and extensions. +* ensuring all permissions are set. + +Scripts for v1 and v2 are provided to accomplish this. +Make sure the application [configuration](configuration.md) matches the values in the script. +Run the script as a super user and check the output carefully to ensure no errors occurred. + +#### PostgreSQL (V1) +Run the SQL script located at `hedera-mirror-importer/src/main/resources/db/scripts/init_v1.sql`. +Edit the file and change the `db_name`, `db_user`, `db_password` `db_owner`, `grpc_user`, or `grpc_password` as appropriate. + +```console +psql postgres -f hedera-mirror-importer/src/main/resources/db/scripts/init_v1.sql +``` + +#### TimescaleDB (V2) +Run the SQL script located at `hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql`. +Edit the file and change the db user names, passwords and schema as appropriate. ```console -psql postgres -f hedera-mirror-importer/src/main/resources/db/scripts/init.sql +psql postgres -f hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql ``` ### Importer @@ -86,7 +101,7 @@ npm test Docker Compose scripts are provided and run all the mirror node components: -- PostgreSQL database +- PostgreSQL/TimescaleDB database - GRPC API - Importer - Monitor @@ -95,14 +110,33 @@ Docker Compose scripts are provided and run all the mirror node components: Containers use the following persisted volumes: - `./db` on your local machine maps to `/var/lib/postgresql/data` in the containers. This contains the files for the - PostgreSQL database. If the database container fails to initialise properly and the database fails to run, you will - have to delete this folder prior to attempting a restart otherwise the database initialisation scripts will not be - run. + PostgreSQL/TimescaleDB database. If the database container fails to initialise properly and the database fails to run, + you will have to delete this folder prior to attempting a restart otherwise the database initialisation scripts will + not be run. - `./data` on your local machine maps to `/var/lib/hedera-mirror-importer` in the container. This contains files downloaded from S3 or GCP. These are necessary not only for the database data to be persisted, but also so that the parsing containers can access file obtained via the downloading containers +### Configuration + +#### TimescaleDB vs PostgreSQL +To utilize the TimescaleDB database over the default PostgreSQL database, disable the PostgreSQL container and enable the TimescaleDB container. + +To achieve this the `docker-compose.yml` can be updated to set the postgres `db` service replicas to 0 whiles removing this same setting from the `timescaledb` service as follows: +```yaml + ... + services: + db: + deploy: + replicas: 0 + ... + timescaledb: + # deploy: + # replicas: 0 + ... +``` + ### Starting Before starting, [configure](configuration.md) the application by updating the [application.yml](../application.yml) diff --git a/docs/operations.md b/docs/operations.md index e6c5c114506..66f105a1bd1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -93,6 +93,49 @@ systemctl status hedera-mirror-importer.service sudo journalctl -fu hedera-mirror-importer.service ``` +### v1 to v2 Data Migration + +To support time series logic the Mirror Node DB schema shifted from PostgeSQL (v1) to TimescaleDB (v2). +[Migrating from a Different PostgreSQL Database](https://docs.timescale.com/latest/getting-started/migrating-data#different-db) highlights the general recommended data migration steps when moving to TimescaleDB. + +For mirror node operators running v1 db schema, the following steps can be taken to upgrade to v2. + +1. Set up a new TimescaleDB database + + A new TimescaleDB server must be spun up. + + Refer to Mirror Node [DB Installation](installation.md#database-setup) for manual instructions. + + To use the Mirror Node configured docker container, simply run: + + ```shell script + $ docker-compose up timescaledb + ``` + + Refer to [TimescaleDB Installation Instructions](https://docs.timescale.com/latest/getting-started/installation) for other installation options. + + > **_NOTE:_** The following steps assume the database, users and schema have been created as detailed above + +2. Configure migration properties + + The configuration file `hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.config` contains db variables for easy running. + These options include variables such as db names, passwords, users, hosts for both the existing db and the new db. + + Update these values appropriately for your db setup. + +3. Run migration script + + From the `hedera-mirror-importer/src/main/resources/db` directory run the `migration.sh` script + ```shell script + $ ./scripts/timescaledb/migration.sh + ``` + + The script uses successive `psql` connections to back up, configure and restore data on the new database nodes. + First it copies over the `flyway_schema_history` table, to maintain migration history. + It then utilizes the migration sql script used by normal flyway operations to create the new tables and then creates the Timescale hypertables based on these. + Following this the tables from the old database are backed up as csv files using `\COPY` and then the data inserted into the new database also using `\COPY`. + Finally the schema of the `flyway_schema_history` is updated and the sequence values are updated to ensure continuation. + ## Monitor The monitor is a Java-based application and should be able to run on any platform that Java supports. That said, we diff --git a/hedera-mirror-grpc/src/test/resources/config/bootstrap.yml b/hedera-mirror-grpc/src/test/resources/config/bootstrap.yml index fa1542274ec..2379c9fdfb2 100644 --- a/hedera-mirror-grpc/src/test/resources/config/bootstrap.yml +++ b/hedera-mirror-grpc/src/test/resources/config/bootstrap.yml @@ -1,7 +1,7 @@ embedded: postgresql: enabled: true - # set to timescaledev/timescaledb-ha:pg12-ts2.0.0-rc3 (same as chart) for v2 db schema. + # set to timescaledev/timescaledb-ha:pg12.5-ts2.0.0-p0 (same as chart) for v2 db schema. docker-image: postgres:9.6-alpine # postgres:12-alpine is current default redis: docker-image: redis:5.0.9-alpine diff --git a/hedera-mirror-importer/src/main/resources/application.yml b/hedera-mirror-importer/src/main/resources/application.yml index 88d3866b88d..f09b4d8b4d6 100644 --- a/hedera-mirror-importer/src/main/resources/application.yml +++ b/hedera-mirror-importer/src/main/resources/application.yml @@ -5,10 +5,13 @@ hedera: host: 127.0.0.1 loadBalance: true name: mirror_node + owner: mirror_node + ownerPassword: mirror_node_pass password: mirror_node_pass port: 5432 restPassword: mirror_api_pass restUsername: mirror_api + schema: public username: mirror_node logging: level: @@ -73,6 +76,7 @@ spring: baselineOnMigrate: true connectRetries: 20 ignoreMissingMigrations: true + password: ${hedera.mirror.importer.db.ownerPassword} placeholders: api-password: ${hedera.mirror.importer.db.restPassword} api-user: ${hedera.mirror.importer.db.restUsername} @@ -81,6 +85,7 @@ spring: compressionAge: 604800000000000 db-name: ${hedera.mirror.importer.db.name} db-user: ${hedera.mirror.importer.db.username} + user: ${hedera.mirror.importer.db.owner} jpa: properties: hibernate: diff --git a/hedera-mirror-importer/src/main/resources/db/migration/v1/V1.33.0__drop_token_account_id.sql b/hedera-mirror-importer/src/main/resources/db/migration/v1/V1.33.0__drop_token_account_id.sql new file mode 100644 index 00000000000..4d02ef3614c --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/migration/v1/V1.33.0__drop_token_account_id.sql @@ -0,0 +1,29 @@ +------------------- +-- Drop token_account id, replacing id primary key with unique index on (created_timestamp) +-- Drop unused functions f_entity_create and encodeentityid +-- Add missing primary keys on file_data and live_hash +------------------- +alter table if exists token_account + drop constraint token_account_pkey; +alter table if exists token_account + add primary key (created_timestamp); +alter table if exists token_account + drop column if exists id; + +drop sequence if exists token_account_id_seq; + +-- drop unused functions +drop function if exists + f_entity_create(bigint, bigint, bigint, integer, bigint, bigint, bigint, bigint, character varying, bytea, bigint, bytea, nanos_timestamp, text); + +drop function if exists + encodeentityid(bigint, bigint, bigint); + +-- add missing primary keys +drop index if exists idx__t_file_data__consensus; +alter table file_data + add primary key (consensus_timestamp); + +drop index if exists idx__t_livehashes__consensus; +alter table live_hash + add primary key (consensus_timestamp); diff --git a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.0__time_scale_init.sql b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.0__time_scale_init.sql index 9d94f8ce822..2b97f07f252 100644 --- a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.0__time_scale_init.sql +++ b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.0__time_scale_init.sql @@ -14,7 +14,7 @@ comment on table account_balance is 'Account balances (historical) in tinybars a create table if not exists account_balance_file ( - consensus_timestamp bigint, + consensus_timestamp bigint not null, count bigint not null, load_start bigint, load_end bigint, @@ -37,7 +37,7 @@ comment on table account_balance_sets is 'Processing state of snapshots of the e -- address_book create table if not exists address_book ( - start_consensus_timestamp bigint, + start_consensus_timestamp bigint not null, end_consensus_timestamp bigint null, file_id bigint not null, node_count int null, diff --git a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.1__hyper_tables.sql b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.1__hyper_tables.sql index 2eb3288b0b9..d0a2ea249da 100644 --- a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.1__hyper_tables.sql +++ b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.1__hyper_tables.sql @@ -1,58 +1,59 @@ ------------------- -- Create hyper tables for tables that have mostly insert logic --- Use default of 604800000000000 ns (7 days) as chunk time interval --- add TIMESTAMPTZ data type column to tables where no monotonically increasing id exists +-- Set chunk_time_interval using parameterized value, usually default of 604800000000000 ns (7 days) +-- Set create_default_indexes to false for tables where a primary key is needed or an index in ASC order is needed. +-- By default TimescaleDB adds an index in DESC order for partitioning column ------------------- -- account_balance -select create_hypertable('account_balance', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('account_balance', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- account_balance_file -select create_hypertable('account_balance_file', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('account_balance_file', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- account_balance_sets -select create_hypertable('account_balance_sets', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('account_balance_sets', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- address_book -select create_hypertable('address_book', 'start_consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('address_book', 'start_consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- address_book_entry -select create_hypertable('address_book_entry', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('address_book_entry', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- contract_result -select create_hypertable('contract_result', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('contract_result', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- crypto_transfer -select create_hypertable('crypto_transfer', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('crypto_transfer', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- file_data -select create_hypertable('file_data', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('file_data', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- live_hash -select create_hypertable('live_hash', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('live_hash', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- non_fee_transfer -select create_hypertable('non_fee_transfer', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('non_fee_transfer', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- record_file -select create_hypertable('record_file', 'consensus_start', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('record_file', 'consensus_start', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- t_application_status hyper table creation skipped as it serves only as a reference table -- t_entities -select create_hypertable('t_entities', 'id', - chunk_time_interval => ${chunkIdInterval}, if_not_exists => true); +select create_hypertable('t_entities', 'id', chunk_time_interval => ${chunkIdInterval}, + create_default_indexes => false, if_not_exists => true); -- t_entity_types hyper table creation skipped as it serves only as a reference table and rarely gets updated @@ -61,25 +62,25 @@ select create_hypertable('t_entities', 'id', -- t_transaction_types hyper table creation skipped as it serves only as a reference table and rarely gets updated -- token -select create_hypertable('token', 'created_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('token', 'created_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- token_account -select create_hypertable('token_account', 'created_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('token_account', 'created_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- token_balance -select create_hypertable('token_balance', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('token_balance', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- token_transfer -select create_hypertable('token_transfer', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('token_transfer', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- topic_message -select create_hypertable('topic_message', 'consensus_timestamp', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('topic_message', 'consensus_timestamp', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); -- transaction -select create_hypertable('transaction', 'consensus_ns', - chunk_time_interval => ${chunkTimeInterval}, if_not_exists => true); +select create_hypertable('transaction', 'consensus_ns', chunk_time_interval => ${chunkTimeInterval}, + create_default_indexes => false, if_not_exists => true); diff --git a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.2__time_scale_index_init.sql b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.2__time_scale_index_init.sql index aae173d72b6..6f5285f827d 100644 --- a/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.2__time_scale_index_init.sql +++ b/hedera-mirror-importer/src/main/resources/db/migration/v2/V2.0.2__time_scale_index_init.sql @@ -4,27 +4,29 @@ -- account_balance alter table account_balance - add constraint account_balance_timestamp_id primary key (consensus_timestamp, account_id); + add primary key (consensus_timestamp, account_id); create index if not exists account_balance__account_timestamp on account_balance (account_id desc, consensus_timestamp desc); -- account_balance_sets alter table account_balance_sets - add constraint account_balance_sets_timestamp primary key (consensus_timestamp); + add primary key (consensus_timestamp); create index if not exists balance_sets__completed on account_balance_sets (is_complete, consensus_timestamp desc); -- account_balance_file +alter table account_balance_file + add primary key (consensus_timestamp); create unique index if not exists account_balance_file__name on account_balance_file (name, consensus_timestamp desc); -alter table account_balance_file - add constraint account_balance_file_timestamp primary key (consensus_timestamp); -- address_book alter table address_book - add constraint address_book_start_timestamp primary key (start_consensus_timestamp); + add primary key (start_consensus_timestamp); -- address_book_entry +alter table address_book_entry + add primary key (consensus_timestamp, memo); create index if not exists address_book_entry__timestamp on address_book_entry (consensus_timestamp); @@ -41,30 +43,32 @@ create index if not exists crypto_transfer__entity_id_consensus_timestamp -- id corresponding to treasury address 0.0.98 -- file_data -create index if not exists file_data__consensus - on file_data (consensus_timestamp desc); +alter table file_data + add primary key (consensus_timestamp); -- live_hash -create index if not exists livehashes__consensus - on live_hash (consensus_timestamp desc); +alter table live_hash + add primary key (consensus_timestamp); -- non_fee_transfer create index if not exists non_fee_transfer__consensus_timestamp on non_fee_transfer (consensus_timestamp); -- record_file +alter table record_file + add primary key (consensus_start); create unique index if not exists record_file_name on record_file (name, consensus_start); -- have to add consensus_start due to partitioning create unique index if not exists record_file_hash on record_file (file_hash, consensus_start); -- have to add consensus_start due to partitioning -create index if not exists record_file__consensus_start - on record_file (consensus_start); create index if not exists record_file__consensus_end on record_file (consensus_end); create index if not exists record_file__prev_hash on record_file (prev_hash); -- t_entities +alter table t_entities + add primary key (id); -- Enforce lowercase hex representation by constraint rather than making indexes on lower(ed25519). alter table t_entities add constraint c__t_entities__lower_ed25519 @@ -73,23 +77,39 @@ create index if not exists entities__ed25519_public_key_hex_natural_id on t_entities (ed25519_public_key_hex, fk_entity_type_id, entity_shard, entity_realm, entity_num); create unique index if not exists entities_unq on t_entities (entity_shard, entity_realm, entity_num, id); --- have to add id due to partitioning +-- have to add id when creating unique indexes due to partitioning + +-- t_entity_types +alter table t_entity_types + add primary key (id); + +-- t_transaction_results +alter table t_transaction_results + add primary key (proto_id); +create unique index if not exists t_transaction_results_name + on t_transaction_results (result); + +-- t_transaction_types +alter table t_transaction_types + add primary key (proto_id); +create unique index if not exists t_transaction_types_name + on t_transaction_types (name); -- token -alter table if exists token - add constraint token_timestamp primary key (created_timestamp); -create index if not exists token_id - on token (token_id); +alter table token + add primary key (created_timestamp); +create unique index if not exists token__id_timestamp + on token (token_id, created_timestamp); -- token_account -alter table if exists token_account - add constraint token_account_timestamp primary key (created_timestamp); -create unique index if not exists token_account__token_account +alter table token_account + add primary key (created_timestamp); +create unique index if not exists token_account__token_account_timestamp on token_account (token_id, account_id, created_timestamp); -- token_balance -alter table if exists token_balance - add constraint token_balance_timestamp_ids primary key (consensus_timestamp, account_id, token_id); +alter table token_balance + add primary key (consensus_timestamp, account_id, token_id); -- token_transfer create index if not exists token_transfer__token_account_timestamp @@ -97,15 +117,17 @@ create index if not exists token_transfer__token_account_timestamp -- topic_message alter table if exists topic_message - add constraint topic_message_timestamp primary key (consensus_timestamp); + add primary key (consensus_timestamp); create index if not exists topic_message__realm_num_timestamp on topic_message (realm_num, topic_num, consensus_timestamp); create unique index if not exists topic_message__topic_num_realm_num_seqnum on topic_message (realm_num, topic_num, sequence_number, consensus_timestamp); --- have to add consensus_timestamp due to partitioning +-- have to add consensus_timestamp when creating unique indexes due to partitioning -- transaction -create index transaction__transaction_id +alter table if exists transaction + add primary key (consensus_ns); +create index if not exists transaction__transaction_id on transaction (valid_start_ns, payer_account_id); -create index transaction__payer_account_id +create index if not exists transaction__payer_account_id on transaction (payer_account_id); diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/drop.sql b/hedera-mirror-importer/src/main/resources/db/scripts/drop.sql index c7334f65413..e6f07e3907c 100644 --- a/hedera-mirror-importer/src/main/resources/db/scripts/drop.sql +++ b/hedera-mirror-importer/src/main/resources/db/scripts/drop.sql @@ -4,7 +4,7 @@ -- drop tables, views, indexes, data types, functions, stored procedures and operators associated with db drop schema if exists public cascade; --- recreate schema used by init.sql +-- recreate schema used by init_v1.sql create schema public; grant usage on schema public to public; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/init.sql b/hedera-mirror-importer/src/main/resources/db/scripts/init_v1.sql similarity index 100% rename from hedera-mirror-importer/src/main/resources/db/scripts/init.sql rename to hedera-mirror-importer/src/main/resources/db/scripts/init_v1.sql diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql b/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql new file mode 100644 index 00000000000..e6dbb18e26b --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/init_v2.sql @@ -0,0 +1,71 @@ +-- init the timescale db mirror node db +-- Change the values below if you are not installing via Docker + +\set db_host 'localhost' +\set db_port 5432 +\set db_name 'mirror_node' +\set db_super_user 'postgres' +\set db_owner 'mirror_node' +\set owner_password 'mirror_node_pass' +\set importer_user 'mirror_importer' +\set importer_password 'mirror_importer_pass' +\set grpc_user 'mirror_grpc' +\set grpc_password 'mirror_grpc_pass' +\set rest_user 'mirror_api' +\set rest_password 'mirror_api_pass' +\set schema_name 'mirrornode' + +-- create owner user +create user :db_owner with login password :'owner_password'; + +-- create primary user and db +create database :db_name with owner :db_owner; + +-- create roles +create role readonly; +create role readwrite in role readonly; + +-- create users +create user :grpc_user with login password :'grpc_password' in role readonly; +create user :rest_user with login password :'rest_password' in role readonly; +create user :importer_user with login password :'importer_password' in role readwrite; + +-- drop timescaledb extension for future install to ensure availability in custom schema +drop extension if exists timescaledb cascade; + +-- connect with db owner to create schema and set schema user permissions +\c :db_name :db_owner +create schema if not exists :schema_name authorization :db_owner; +grant usage on schema :schema_name to public; + +-- revoke default public permissions on schema +revoke create on schema :schema_name from public; + +-- grant connect and schema access to readonly role +grant connect on database :db_name to readonly; +grant usage on schema :schema_name to readonly; + +-- grant select privileges on tables to readonly +grant select on all tables in schema :schema_name to readonly; +alter default privileges in schema :schema_name grant select on tables to readonly; + +-- grant select privileges on sequences to readonly +grant select on all sequences in schema :schema_name to readonly; +alter default privileges in schema :schema_name grant select on sequences to readonly; + +-- grant write privileges on sequences to readwrite +grant insert, update on all tables in schema :schema_name to readwrite; +alter default privileges in schema :schema_name grant insert, update on tables to readwrite; +grant usage on all sequences in schema :schema_name to readwrite; +alter default privileges in schema :schema_name grant usage on sequences to readwrite; + +-- alter search path for given schema as super user +\c :db_name :db_super_user +alter user :db_owner set search_path = :schema_name, public; +alter user :importer_user set search_path = :schema_name, public; +alter user :grpc_user set search_path = :schema_name, public; +alter user :rest_user set search_path = :schema_name, public; + +-- add extensions, ensuring they're available to new schema +create extension if not exists timescaledb cascade schema :schema_name; +create extension if not exists pg_stat_statements cascade schema :schema_name; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/alterSchema.sql b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/alterSchema.sql new file mode 100644 index 00000000000..81b061b1fbc --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/alterSchema.sql @@ -0,0 +1,13 @@ +------------------- +-- alter tables by removing domains +-- update to custom schema +------------------- + +\set newSchema mirrornode +-- Update schema from public to custom schema e.g mirrornode +alter table flyway_schema_history + set schema :newSchema; + +-- update sequence start values +select setval('address_book_entry_id_seq', (select max(id) from address_book_entry)); +select setval('record_file_id_seq', (select max(id) from record_file)); diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvBackupTables.sql b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvBackupTables.sql new file mode 100644 index 00000000000..9e344d3dd0f --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvBackupTables.sql @@ -0,0 +1,47 @@ +------------------- +-- Backup tables use efficient COPY process to CSV's +------------------- + +\copy account_balance to account_balance.csv delimiter ',' csv; + +\copy account_balance_file to account_balance_file.csv delimiter ',' csv; + +\copy account_balance_sets to account_balance_sets.csv delimiter ',' csv; + +\copy address_book to address_book.csv delimiter ',' csv; + +\copy address_book_entry to address_book_entry.csv delimiter ',' csv; + +\copy contract_result to contract_result.csv delimiter ',' csv; + +\copy crypto_transfer to crypto_transfer.csv delimiter ',' csv; + +\copy file_data to file_data.csv delimiter ',' csv; + +\copy live_hash to live_hash.csv delimiter ',' csv; + +\copy non_fee_transfer to non_fee_transfer.csv delimiter ',' csv; + +\copy record_file to record_file.csv delimiter ',' csv; + +\copy t_application_status to t_application_status.csv delimiter ',' csv; + +\copy t_entities to t_entities.csv delimiter ',' csv; + +\copy t_entity_types to t_entity_types.csv delimiter ',' csv; + +\copy t_transaction_results to t_transaction_results.csv delimiter ',' csv; + +\copy t_transaction_types to t_transaction_types.csv delimiter ',' csv; + +\copy token to token.csv delimiter ',' csv; + +\copy token_account to token_account.csv delimiter ',' csv; + +\copy token_balance to token_balance.csv delimiter ',' csv; + +\copy token_transfer to token_transfer.csv delimiter ',' csv; + +\copy topic_message to topic_message.csv delimiter ',' csv; + +\copy transaction to transaction.csv delimiter ',' csv; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvRestoreTables.sql b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvRestoreTables.sql new file mode 100644 index 00000000000..dc77e28c6c9 --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/csvRestoreTables.sql @@ -0,0 +1,47 @@ +------------------- +-- Restore Backup tables use efficient COPY process to CSV's +------------------- + +\copy account_balance from account_balance.csv csv; + +\copy account_balance_file from account_balance_file.csv csv; + +\copy account_balance_sets from account_balance_sets.csv csv; + +\copy address_book from address_book.csv csv; + +\copy address_book_entry from address_book_entry.csv csv; + +\copy contract_result from contract_result.csv csv; + +\copy crypto_transfer from crypto_transfer.csv csv; + +\copy file_data from file_data.csv csv; + +\copy live_hash from live_hash.csv csv; + +\copy non_fee_transfer from non_fee_transfer.csv csv; + +\copy record_file from record_file.csv csv; + +\copy t_application_status from t_application_status.csv csv; + +\copy t_entities from t_entities.csv csv; + +\copy t_entity_types from t_entity_types.csv csv; + +\copy t_transaction_results from t_transaction_results.csv csv; + +\copy t_transaction_types from t_transaction_types.csv csv; + +\copy token from token.csv csv; + +\copy token_account from token_account.csv csv; + +\copy token_balance from token_balance.csv csv; + +\copy token_transfer from token_transfer.csv csv; + +\copy topic_message from topic_message.csv csv; + +\copy transaction from transaction.csv csv; diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.config b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.config new file mode 100644 index 00000000000..70f0d71dbf8 --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.config @@ -0,0 +1,12 @@ +OLD_DB_HOST=localhost +OLD_DB_NAME=mirror_node +OLD_DB_PORT=5432 +OLD_DB_USER=mirror_node +OLD_PASSWORD=mirror_node_pass +NEW_DB_HOST=localhost +NEW_DB_NAME=mirror_node +NEW_DB_PORT=6432 +NEW_DB_USER=mirror_node +NEW_PASSWORD=mirror_node_pass +CHUNK_INTERVAL_TIME=604800000000000 +CHUNK_INTERVAL_ID=10000 diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.sh b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.sh new file mode 100755 index 00000000000..755f680b77f --- /dev/null +++ b/hedera-mirror-importer/src/main/resources/db/scripts/timescaledb/migration.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +echo "BASH_VERSION: $BASH_VERSION" +set -e + +. scripts/timescaledb/migration.config + +if [[ -z $OLD_DB_HOST ]]; then + echo "Old host name is not set. Please configure OLD_DB_HOST in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $OLD_DB_NAME ]]; then + echo "Old db name is not set. Please configure OLD_DB_NAME in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $OLD_DB_PORT ]]; then + echo "Old port is not set. Please configure OLD_DB_PORT in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $OLD_DB_USER ]]; then + echo "Old db primary user name is not set. Please configure OLD_DB_USER in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $OLD_PASSWORD ]]; then + echo "Old db primary user password is not set. Please configure OLD_PASSWORD in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $NEW_DB_HOST ]]; then + echo "New host name is not set. Please configure NEW_DB_HOST in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $NEW_DB_NAME ]]; then + echo "New db name is not set. Please configure NEW_DB_NAME in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $NEW_DB_PORT ]]; then + echo "New db port is not set. Please configure NEW_DB_PORT in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $NEW_DB_USER ]]; then + echo "New db primary user name is not set. Please configure NEW_DB_USER in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $NEW_PASSWORD ]]; then + echo "New db primary user password is not set. Please configure NEW_PASSWORD in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $CHUNK_INTERVAL_TIME ]]; then + echo "New db chunk interval time is not set. Please configure CHUNK_INTERVAL_TIME in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +if [[ -z $CHUNK_INTERVAL_ID ]]; then + echo "New db chunk interval id is not set. Please configure CHUNK_INTERVAL_ID in migration.config file and rerun './scripts/timescaledb/migration.sh'" + exit 1 +fi + +start_time="$(date -u +%s)" +# assumes 1. Valid populated old mirror node postgres db with appropriate user 2. New empty TimescaleDB db host with appropriate user +echo "Migrating Mirror Node Data from Postgres($OLD_DB_HOST:$OLD_DB_PORT) to TimescaleDB($NEW_DB_HOST:$NEW_DB_PORT)..." + +echo "1. Backing up flyway table schema from Postgres($OLD_DB_HOST:$OLD_DB_PORT)..." +PGPASSWORD=${OLD_PASSWORD} pg_dump -h $OLD_DB_HOST -p $OLD_DB_PORT -U $OLD_DB_USER --table public.flyway_schema_history -f mirror_node_${start_time}.bak mirror_node + +echo "2. Restoring flyway_schema_history to TimescaleDB($NEW_DB_HOST:$NEW_DB_PORT)..." +PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER scripts/timescaledb/createHyperTables.sql +PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER -f scripts/timescaledb/createHyperTables.sql + +echo "5. Backing up tables from from Postgres($OLD_DB_HOST:$OLD_DB_PORT) to separate CSV's..." +PGPASSWORD=${OLD_PASSWORD} psql -h $OLD_DB_HOST -d $OLD_DB_NAME -p $OLD_DB_PORT -U $OLD_DB_USER -f scripts/timescaledb/csvBackupTables.sql + +## Optionally use https://github.com/timescale/timescaledb-parallel-copy as it's mulithreaded +echo "6. Restoring CSV backups to TimescaleDB($NEW_DB_HOST:$NEW_DB_PORT)..." +PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER -f scripts/timescaledb/csvRestoreTables.sql + +echo "7. Alter schema on TimescaleDB($NEW_DB_HOST:$NEW_DB_PORT) to support improved format..." +PGPASSWORD=${NEW_PASSWORD} psql -h $NEW_DB_HOST -d $NEW_DB_NAME -p $NEW_DB_PORT -U $NEW_DB_USER -f scripts/timescaledb/alterSchema.sql + +# leave index creation and policy sets to migration 2.0 +end_time="$(date -u +%s)" + +elapsed="$(($end_time - $start_time))" +echo "Migration from postgres to timescale took a total of $elapsed seconds" diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/AbstractEntityRecordItemListenerTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/AbstractEntityRecordItemListenerTest.java index 574a41e1012..bf9ea955118 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/AbstractEntityRecordItemListenerTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/AbstractEntityRecordItemListenerTest.java @@ -180,8 +180,9 @@ protected final void assertContract(ContractID contractId, Entities dbEntity) { protected void parseRecordItemAndCommit(RecordItem recordItem) { String fileName = UUID.randomUUID().toString(); EntityId nodeAccountId = EntityId.of(TestUtils.toAccountId("0.0.3")); - RecordFile recordFile = new RecordFile(0L, 0L, null, fileName, 0L, 0L, UUID.randomUUID() - .toString(), "", nodeAccountId, 0L, 0); + RecordFile recordFile = new RecordFile(recordItem.getConsensusTimestamp(), + recordItem.getConsensusTimestamp() + 1, null, fileName, 0L, 0L, + UUID.randomUUID().toString(), "", nodeAccountId, 0L, 0); recordFileRepository.save(recordFile); recordStreamFileListener.onStart(new StreamFileData(fileName, null)); // open connection entityRecordItemListener.onItem(recordItem); diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/sql/SqlEntityListenerTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/sql/SqlEntityListenerTest.java index f10527acb22..ba5f36d2d13 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/sql/SqlEntityListenerTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/record/entity/sql/SqlEntityListenerTest.java @@ -94,7 +94,7 @@ public class SqlEntityListenerTest extends IntegrationTest { @BeforeEach final void beforeEach() { String newFileHash = UUID.randomUUID().toString(); - recordFile = insertRecordFileRecord(fileName, newFileHash, "fileHash0"); + recordFile = insertRecordFileRecord(fileName, newFileHash, "fileHash0", 1L); sqlEntityListener.onStart(new StreamFileData(fileName, null)); } @@ -240,7 +240,7 @@ void onEntityIdDuplicates() throws Exception { sqlEntityListener.onEntityId(entityId); // duplicate within file completeFileAndCommit(); - recordFile = insertRecordFileRecord(UUID.randomUUID().toString(), null, null); + recordFile = insertRecordFileRecord(UUID.randomUUID().toString(), null, null, 1L); sqlEntityListener.onStart(new StreamFileData(fileName, null)); sqlEntityListener.onEntityId(entityId); // duplicate across files completeFileAndCommit(); @@ -341,7 +341,7 @@ private String completeFileAndCommit() { return recordFile.getFileHash(); } - private RecordFile insertRecordFileRecord(String filename, String fileHash, String prevHash) { + private RecordFile insertRecordFileRecord(String filename, String fileHash, String prevHash, long consensusStart) { if (fileHash == null) { fileHash = UUID.randomUUID().toString(); } @@ -350,7 +350,8 @@ private RecordFile insertRecordFileRecord(String filename, String fileHash, Stri } EntityId nodeAccountId = EntityId.of(TestUtils.toAccountId("0.0.3")); - RecordFile rf = new RecordFile(1L, 2L, null, filename, 0L, 0L, fileHash, prevHash, nodeAccountId, 0L, 0); + RecordFile rf = new RecordFile(consensusStart, consensusStart + 1, null, filename, 0L, 0L, fileHash, prevHash + , nodeAccountId, 0L, 0); recordFileRepository.save(rf); return rf; } diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/repository/AddressBookEntryRepositoryTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/repository/AddressBookEntryRepositoryTest.java index 2d790ca3456..21bda0151c6 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/repository/AddressBookEntryRepositoryTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/repository/AddressBookEntryRepositoryTest.java @@ -23,8 +23,8 @@ public class AddressBookEntryRepositoryTest extends AbstractRepositoryTest { @Test void save() { - addressBookRepository.save(addressBook(null)); - AddressBookEntry addressBookEntry = addressBookEntryRepository.save(addressBookEntry(null)); + addressBookRepository.save(addressBook(null, 1L)); + AddressBookEntry addressBookEntry = addressBookEntryRepository.save(addressBookEntry(null, 1L, 3)); assertThat(addressBookEntryRepository.findById(addressBookEntry.getId())) .get() .isEqualTo(addressBookEntry); @@ -32,24 +32,26 @@ void save() { @Test void verifySequence() { - addressBookRepository.save(addressBook(null)); - addressBookEntryRepository.save(addressBookEntry(null)); - addressBookEntryRepository.save(addressBookEntry(null)); - addressBookEntryRepository.save(addressBookEntry(null)); + long consensusTimestamp = 1L; + addressBookRepository.save(addressBook(null, consensusTimestamp)); + addressBookEntryRepository.save(addressBookEntry(null, consensusTimestamp, 3)); + addressBookEntryRepository.save(addressBookEntry(null, consensusTimestamp, 4)); + addressBookEntryRepository.save(addressBookEntry(null, consensusTimestamp, 5)); assertThat(addressBookEntryRepository.findAll()) .isNotNull() .extracting(AddressBookEntry::getId) .containsSequence(1L, 2L, 3L); } - private AddressBookEntry addressBookEntry(Consumer nodeAddressCustomizer) { + private AddressBookEntry addressBookEntry(Consumer nodeAddressCustomizer, long consensusTimestamp, long nodeAccountId) { + String nodeAccountIdString = String.format("0.0.%s", nodeAccountId); AddressBookEntry.AddressBookEntryBuilder builder = AddressBookEntry.builder() - .consensusTimestamp(0L) + .consensusTimestamp(consensusTimestamp) .ip("127.0.0.1") .publicKey("rsa+public/key") - .memo("0.0.3") - .nodeAccountId(EntityId.of("0.0.5", EntityTypeEnum.ACCOUNT)) - .nodeId(5L) + .memo(nodeAccountIdString) + .nodeAccountId(EntityId.of(nodeAccountIdString, EntityTypeEnum.ACCOUNT)) + .nodeId(nodeAccountId) .nodeCertHash("nodeCertHash".getBytes()); if (nodeAddressCustomizer != null) { @@ -59,10 +61,11 @@ private AddressBookEntry addressBookEntry(Consumer addressBookCustomizer) { + private AddressBook addressBook(Consumer addressBookCustomizer, + long consensusTimestamp) { AddressBook.AddressBookBuilder builder = AddressBook.builder() - .startConsensusTimestamp(0L) + .startConsensusTimestamp(consensusTimestamp) .fileData("address book memo".getBytes()) .fileId(addressBookEntityId102); diff --git a/hedera-mirror-importer/src/test/resources/config/application.yml b/hedera-mirror-importer/src/test/resources/config/application.yml index f3c154f0398..ffc88771945 100644 --- a/hedera-mirror-importer/src/test/resources/config/application.yml +++ b/hedera-mirror-importer/src/test/resources/config/application.yml @@ -22,6 +22,9 @@ hedera: startDate: 1970-01-01T00:00:00Z spring: + flyway: + password: ${hedera.mirror.importer.db.password} + user: ${hedera.mirror.importer.db.username} redis: url: redis://${embedded.redis.user}:${embedded.redis.password}@${embedded.redis.host}:${embedded.redis.port} task: diff --git a/hedera-mirror-importer/src/test/resources/config/bootstrap.yml b/hedera-mirror-importer/src/test/resources/config/bootstrap.yml index f842a712d4b..5bbfc4a187d 100644 --- a/hedera-mirror-importer/src/test/resources/config/bootstrap.yml +++ b/hedera-mirror-importer/src/test/resources/config/bootstrap.yml @@ -5,7 +5,7 @@ embedded: # so it is enabled only for those tests. enabled: false postgresql: - # set to timescaledev/timescaledb-ha:pg12-ts2.0.0-rc3 (same as chart) for v2 db schema. + # set to timescaledev/timescaledb-ha:pg12.5-ts2.0.0-p0 (same as chart) for v2 db schema. docker-image: postgres:9.6-alpine redis: docker-image: redis:5.0.9-alpine diff --git a/hedera-mirror-rest/__tests__/integrationDbOps.js b/hedera-mirror-rest/__tests__/integrationDbOps.js index 3b68ae12b1c..f0af0e90e79 100644 --- a/hedera-mirror-rest/__tests__/integrationDbOps.js +++ b/hedera-mirror-rest/__tests__/integrationDbOps.js @@ -50,7 +50,7 @@ const v1SchemaConfigs = { const v2SchemaConfigs = { docker: { imageName: 'timescaledev/timescaledb-ha', - tagName: 'pg12-ts2.0.0-rc3', + tagName: 'pg12.5-ts2.0.0-p0', }, flyway: { baselineVersion: '1.999.999',