Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: use latest complete snapshot #68

Merged
merged 2 commits into from
Jul 11, 2024
Merged

chore: use latest complete snapshot #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c098e23
chore: determine latest complete snapshot by bundle summary
troykessler Jul 2, 2024
f8e103e
fix: get url with backoff
troykessler Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions statesync/helpers/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"github.com/KYVENetwork/ksync/collectors/bundles"
"github.com/KYVENetwork/ksync/collectors/pool"
"github.com/KYVENetwork/ksync/utils"
"strconv"
"strings"
)

// GetSnapshotPoolHeight returns the height of the snapshot the pool is currently archiving.
Expand Down Expand Up @@ -77,6 +79,18 @@ func GetSnapshotBoundaries(restEndpoint string, poolId int64) (startHeight int64
// that the snapshot before the current one has to be completed (- (chunkIndex +1))
highestUsableSnapshotBundleId := poolResponse.Pool.Data.TotalBundles - 1 - (chunkIndex + 1)

// check if the current chunk is the last chunk of the snapshot, if yes we highestUsableSnapshotBundleId
// is the current one. we check this by comparing the number of chunks from the bundle summary with the
// current chunk index from the current key.
summary := strings.Split(poolResponse.Pool.Data.CurrentSummary, "/")
// bundle summary format is "height/format/chunkIndex/chunks"
// if the summary does not have the right format we skip because this is probably a legacy bundle summary.
if len(summary) == 4 {
if chunks, _ := strconv.ParseInt(summary[3], 10, 64); chunks == chunkIndex+1 {
highestUsableSnapshotBundleId = poolResponse.Pool.Data.TotalBundles - 1
}
}

bundle, err := bundles.GetFinalizedBundleById(restEndpoint, poolId, highestUsableSnapshotBundleId)
if err != nil {
return startHeight, endHeight, fmt.Errorf("failed to get finalized bundle with id %d: %w", highestUsableSnapshotBundleId, err)
Expand Down
11 changes: 6 additions & 5 deletions types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ type PoolResponse = struct {
Pool struct {
Id int64 `json:"id"`
Data struct {
Runtime string `json:"runtime"`
StartKey string `json:"start_key"`
CurrentKey string `json:"current_key"`
TotalBundles int64 `json:"total_bundles"`
Config string `json:"config"`
Runtime string `json:"runtime"`
StartKey string `json:"start_key"`
CurrentKey string `json:"current_key"`
CurrentSummary string `json:"current_summary"`
TotalBundles int64 `json:"total_bundles"`
Config string `json:"config"`
} `json:"data"`
} `json:"pool"`
}
Expand Down
2 changes: 1 addition & 1 deletion utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func GetFromUrlWithOptions(url string, options GetFromUrlOptions) ([]byte, error

// GetFromUrlWithBackoff tries to fetch data from url with exponential backoff
func GetFromUrlWithBackoff(url string) (data []byte, err error) {
return GetFromUrlWithOptions(url, GetFromUrlOptions{SkipTLSVerification: true})
return GetFromUrlWithOptions(url, GetFromUrlOptions{SkipTLSVerification: true, WithBackoff: true})
}

func CreateSha256Checksum(input []byte) (hash string) {
Expand Down