From 0e38be5610164a7fe40617e5549db70fdea2afc2 Mon Sep 17 00:00:00 2001 From: Artem Poltorzhitskiy Date: Tue, 22 Jun 2021 17:10:12 +0300 Subject: [PATCH] Improvement: refresh materialized view concurrently (#705) --- cmd/indexer/indexer/boost.go | 2 +- cmd/indexer/scripts/head_stats_view.sql | 4 +++- .../scripts/series_consumed_gas_by_month_view.sql | 4 +++- cmd/indexer/scripts/series_contract_by_month_view.sql | 4 +++- cmd/indexer/scripts/series_operation_by_month_view.sql | 4 +++- .../scripts/series_paid_storage_diff_by_month_view.sql | 4 +++- cmd/metrics/time-based.go | 10 +++++----- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go index 0ef29c417..b1517102c 100644 --- a/cmd/indexer/indexer/boost.go +++ b/cmd/indexer/indexer/boost.go @@ -154,7 +154,7 @@ func executeScripts(db *core.Postgres, network types.Network) error { script := string(raw) if strings.HasPrefix(files[i].Name(), "series_") { - script = fmt.Sprintf(script, network.String(), network) + script = fmt.Sprintf(script, network.String(), network, network.String(), network.String()) } if err := db.Execute(script); err != nil { diff --git a/cmd/indexer/scripts/head_stats_view.sql b/cmd/indexer/scripts/head_stats_view.sql index 941ab5c94..2012c5d26 100644 --- a/cmd/indexer/scripts/head_stats_view.sql +++ b/cmd/indexer/scripts/head_stats_view.sql @@ -5,4 +5,6 @@ select network, count(*) as value, 'contracts_count' as stats_type from contract union all select network, count(distinct(hash)) as value, 'unique_contracts_count' as stats_type from contracts group by network union all -select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network \ No newline at end of file +select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network; + +create unique index if not exists head_stats_idx on head_stats(network, stats_type); \ No newline at end of file diff --git a/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql b/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql index f24659833..bf4a5ff11 100644 --- a/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql +++ b/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql @@ -12,4 +12,6 @@ select from f left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d)) group by 1 -order by date_part \ No newline at end of file +order by date_part; + +create unique index if not exists series_consumed_gas_by_month_%s_idx on series_consumed_gas_by_month_%s(date_part); \ No newline at end of file diff --git a/cmd/indexer/scripts/series_contract_by_month_view.sql b/cmd/indexer/scripts/series_contract_by_month_view.sql index 10aa5335e..b35b0d02d 100644 --- a/cmd/indexer/scripts/series_contract_by_month_view.sql +++ b/cmd/indexer/scripts/series_contract_by_month_view.sql @@ -12,4 +12,6 @@ select from f left join contracts on date_trunc('month', contracts.timestamp) = f.val where ((network = %d)) group by 1 -order by date_part \ No newline at end of file +order by date_part; + +create unique index if not exists series_contract_by_month_%s_idx on series_contract_by_month_%s(date_part); \ No newline at end of file diff --git a/cmd/indexer/scripts/series_operation_by_month_view.sql b/cmd/indexer/scripts/series_operation_by_month_view.sql index 58d4c9c4b..72a5af243 100644 --- a/cmd/indexer/scripts/series_operation_by_month_view.sql +++ b/cmd/indexer/scripts/series_operation_by_month_view.sql @@ -12,4 +12,6 @@ select from f left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d) and (entrypoint is not null and entrypoint != '') and (status = 1)) group by 1 -order by date_part \ No newline at end of file +order by date_part; + +create unique index if not exists series_operation_by_month_%s_idx on series_operation_by_month_%s(date_part); \ No newline at end of file diff --git a/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql b/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql index 2e45feb24..6cc329302 100644 --- a/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql +++ b/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql @@ -12,4 +12,6 @@ select from f left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d)) group by 1 -order by date_part \ No newline at end of file +order by date_part; + +create unique index if not exists series_paid_storage_size_diff_by_month_%s_idx on series_paid_storage_size_diff_by_month_%s(date_part); \ No newline at end of file diff --git a/cmd/metrics/time-based.go b/cmd/metrics/time-based.go index 05c2ed7c3..fab427d0a 100644 --- a/cmd/metrics/time-based.go +++ b/cmd/metrics/time-based.go @@ -29,7 +29,7 @@ func timeBasedTask(period time.Duration, handler func() error, closeChan chan st func (ctx *Context) updateMaterializedViews() error { return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error { - return tx.Exec(`REFRESH MATERIALIZED VIEW head_stats;`).Error + return tx.Exec(`REFRESH MATERIALIZED VIEW CONCURRENTLY head_stats;`).Error }) } @@ -37,19 +37,19 @@ func (ctx *Context) updateSeriesMaterializedViews() error { return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error { for network := range ctx.Config.Indexer.Networks { - if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_contract_by_month_%s;`, network)).Error; err != nil { + if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_contract_by_month_%s;`, network)).Error; err != nil { return err } - if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_operation_by_month_%s;`, network)).Error; err != nil { + if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_operation_by_month_%s;`, network)).Error; err != nil { return err } - if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil { + if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil { return err } - if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_consumed_gas_by_month_%s;`, network)).Error; err != nil { + if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_consumed_gas_by_month_%s;`, network)).Error; err != nil { return err } }