Improvement: refresh materialized view concurrently (#705)

baking-bad · Jun 22, 2021 · 0e38be5 · 0e38be5
1 parent a1c18f9
commit 0e38be5
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 11 deletions.
diff --git a/cmd/indexer/indexer/boost.go b/cmd/indexer/indexer/boost.go
@@ -154,7 +154,7 @@ func executeScripts(db *core.Postgres, network types.Network) error {
 
 		script := string(raw)
 		if strings.HasPrefix(files[i].Name(), "series_") {
-			script = fmt.Sprintf(script, network.String(), network)
+			script = fmt.Sprintf(script, network.String(), network, network.String(), network.String())
 		}
 
 		if err := db.Execute(script); err != nil {

diff --git a/cmd/indexer/scripts/head_stats_view.sql b/cmd/indexer/scripts/head_stats_view.sql
@@ -5,4 +5,6 @@ select network, count(*) as value, 'contracts_count' as stats_type from contract
 union all
 select network, count(distinct(hash)) as value, 'unique_contracts_count' as stats_type from contracts group by network
 union all
-select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network
+select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network;
+
+create unique index if not exists head_stats_idx on head_stats(network, stats_type);
diff --git a/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql b/cmd/indexer/scripts/series_consumed_gas_by_month_view.sql
@@ -12,4 +12,6 @@ select
 from f
 left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d))
 group by 1
-order by date_part
+order by date_part;
+
+create unique index if not exists series_consumed_gas_by_month_%s_idx on series_consumed_gas_by_month_%s(date_part);
diff --git a/cmd/indexer/scripts/series_contract_by_month_view.sql b/cmd/indexer/scripts/series_contract_by_month_view.sql
@@ -12,4 +12,6 @@ select
 from f
 left join contracts on date_trunc('month', contracts.timestamp) = f.val where ((network = %d))
 group by 1
-order by date_part
+order by date_part;
+
+create unique index if not exists series_contract_by_month_%s_idx on series_contract_by_month_%s(date_part);
diff --git a/cmd/indexer/scripts/series_operation_by_month_view.sql b/cmd/indexer/scripts/series_operation_by_month_view.sql
@@ -12,4 +12,6 @@ select
 from f
 left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d) and (entrypoint is not null and entrypoint != '') and (status = 1))
 group by 1
-order by date_part
+order by date_part;
+
+create unique index if not exists series_operation_by_month_%s_idx on series_operation_by_month_%s(date_part);
diff --git a/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql b/cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql
@@ -12,4 +12,6 @@ select
 from f
 left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d))
 group by 1
-order by date_part
+order by date_part;
+
+create unique index if not exists series_paid_storage_size_diff_by_month_%s_idx on series_paid_storage_size_diff_by_month_%s(date_part);
diff --git a/cmd/metrics/time-based.go b/cmd/metrics/time-based.go
@@ -29,27 +29,27 @@ func timeBasedTask(period time.Duration, handler func() error, closeChan chan st
 
 func (ctx *Context) updateMaterializedViews() error {
 	return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
-		return tx.Exec(`REFRESH MATERIALIZED VIEW head_stats;`).Error
+		return tx.Exec(`REFRESH MATERIALIZED VIEW CONCURRENTLY head_stats;`).Error
 	})
 }
 
 func (ctx *Context) updateSeriesMaterializedViews() error {
 	return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
 		for network := range ctx.Config.Indexer.Networks {
 
-			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_contract_by_month_%s;`, network)).Error; err != nil {
+			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_contract_by_month_%s;`, network)).Error; err != nil {
 				return err
 			}
 
-			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_operation_by_month_%s;`, network)).Error; err != nil {
+			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_operation_by_month_%s;`, network)).Error; err != nil {
 				return err
 			}
 
-			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil {
+			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil {
 				return err
 			}
 
-			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_consumed_gas_by_month_%s;`, network)).Error; err != nil {
+			if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_consumed_gas_by_month_%s;`, network)).Error; err != nil {
 				return err
 			}
 		}