Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improvement: refresh materialized view concurrently #705

Merged
merged 1 commit into from
Jun 22, 2021
Merged

Improvement: refresh materialized view concurrently #705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/indexer/indexer/boost.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ func executeScripts(db *core.Postgres, network types.Network) error {

script := string(raw)
if strings.HasPrefix(files[i].Name(), "series_") {
script = fmt.Sprintf(script, network.String(), network)
script = fmt.Sprintf(script, network.String(), network, network.String(), network.String())
}

if err := db.Execute(script); err != nil {
Expand Down
4 changes: 3 additions & 1 deletion cmd/indexer/scripts/head_stats_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,6 @@ select network, count(*) as value, 'contracts_count' as stats_type from contract
union all
select network, count(distinct(hash)) as value, 'unique_contracts_count' as stats_type from contracts group by network
union all
select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network
select network, count(*) as value, 'fa_count' as stats_type from contracts where (tags & 448) > 0 group by network;

create unique index if not exists head_stats_idx on head_stats(network, stats_type);
4 changes: 3 additions & 1 deletion cmd/indexer/scripts/series_consumed_gas_by_month_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ select
from f
left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d))
group by 1
order by date_part
order by date_part;

create unique index if not exists series_consumed_gas_by_month_%s_idx on series_consumed_gas_by_month_%s(date_part);
4 changes: 3 additions & 1 deletion cmd/indexer/scripts/series_contract_by_month_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ select
from f
left join contracts on date_trunc('month', contracts.timestamp) = f.val where ((network = %d))
group by 1
order by date_part
order by date_part;

create unique index if not exists series_contract_by_month_%s_idx on series_contract_by_month_%s(date_part);
4 changes: 3 additions & 1 deletion cmd/indexer/scripts/series_operation_by_month_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ select
from f
left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d) and (entrypoint is not null and entrypoint != '') and (status = 1))
group by 1
order by date_part
order by date_part;

create unique index if not exists series_operation_by_month_%s_idx on series_operation_by_month_%s(date_part);
4 changes: 3 additions & 1 deletion cmd/indexer/scripts/series_paid_storage_diff_by_month_view.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ select
from f
left join operations on date_trunc('month', operations.timestamp) = f.val where ((network = %d))
group by 1
order by date_part
order by date_part;

create unique index if not exists series_paid_storage_size_diff_by_month_%s_idx on series_paid_storage_size_diff_by_month_%s(date_part);
10 changes: 5 additions & 5 deletions cmd/metrics/time-based.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,27 +29,27 @@ func timeBasedTask(period time.Duration, handler func() error, closeChan chan st

func (ctx *Context) updateMaterializedViews() error {
return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
return tx.Exec(`REFRESH MATERIALIZED VIEW head_stats;`).Error
return tx.Exec(`REFRESH MATERIALIZED VIEW CONCURRENTLY head_stats;`).Error
})
}

func (ctx *Context) updateSeriesMaterializedViews() error {
return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
for network := range ctx.Config.Indexer.Networks {

if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_contract_by_month_%s;`, network)).Error; err != nil {
if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_contract_by_month_%s;`, network)).Error; err != nil {
return err
}

if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_operation_by_month_%s;`, network)).Error; err != nil {
if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_operation_by_month_%s;`, network)).Error; err != nil {
return err
}

if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil {
if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_paid_storage_size_diff_by_month_%s;`, network)).Error; err != nil {
return err
}

if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW series_consumed_gas_by_month_%s;`, network)).Error; err != nil {
if err := tx.Exec(fmt.Sprintf(`REFRESH MATERIALIZED VIEW CONCURRENTLY series_consumed_gas_by_month_%s;`, network)).Error; err != nil {
return err
}
}
Expand Down