From 83b3c2f1fa439a4552a490cfd627efe6e37e381d Mon Sep 17 00:00:00 2001 From: Antonio Navarro Perez Date: Mon, 30 Sep 2024 13:30:10 +0200 Subject: [PATCH] feat: Store block timestamps. (#44) --- backup/backup.go | 17 ++++++++-------- backup/backup_test.go | 28 ++++++++++++++++++++------ backup/client/client.go | 17 ++++++++++++---- backup/client/http/http.go | 13 +++++++++--- backup/mock_test.go | 14 +++++++------ restore/restore_test.go | 41 +++++++++++++++++++++++++++++++++++++- types/types.go | 7 ++++++- 7 files changed, 108 insertions(+), 29 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 8d2def2..1d08049 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -6,13 +6,13 @@ import ( "fmt" "time" + _ "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/tx-archive/backup/client" "github.com/gnolang/tx-archive/backup/writer" "github.com/gnolang/tx-archive/log" "github.com/gnolang/tx-archive/log/noop" "github.com/gnolang/tx-archive/types" - - _ "github.com/gnolang/gno/gno.land/pkg/sdk/vm" ) // Service is the chain backup service @@ -56,22 +56,23 @@ func (s *Service) ExecuteBackup(ctx context.Context, cfg Config) error { // Keep track of total txs backed up totalTxs := uint64(0) - fetchAndWrite := func(block uint64) error { - txs, txErr := s.client.GetBlockTransactions(block) + fetchAndWrite := func(height uint64) error { + block, txErr := s.client.GetBlock(height) if txErr != nil { return fmt.Errorf("unable to fetch block transactions, %w", txErr) } // Skip empty blocks - if len(txs) == 0 { + if len(block.Txs) == 0 { return nil } // Save the block transaction data, if any - for _, tx := range txs { + for _, tx := range block.Txs { data := &types.TxData{ - Tx: tx, - BlockNum: block, + Tx: tx, + BlockNum: block.Height, + Timestamp: block.Timestamp, } // Write the tx data to the file diff --git a/backup/backup_test.go b/backup/backup_test.go index 46ab293..b77d5f4 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -10,11 +10,13 @@ import ( "github.com/gnolang/gno/tm2/pkg/amino" "github.com/gnolang/gno/tm2/pkg/std" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gnolang/tx-archive/backup/client" "github.com/gnolang/tx-archive/backup/writer/standard" "github.com/gnolang/tx-archive/log/noop" "github.com/gnolang/tx-archive/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestBackup_DetermineRightBound(t *testing.T) { @@ -96,17 +98,23 @@ func TestBackup_ExecuteBackup_FixedRange(t *testing.T) { cfg = DefaultConfig() + blockTime = time.Date(1987, 0o6, 24, 6, 32, 11, 0, time.FixedZone("Europe/Madrid", 0)) + mockClient = &mockClient{ getLatestBlockNumberFn: func() (uint64, error) { return toBlock, nil }, - getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) { + getBlockFn: func(blockNum uint64) (*client.Block, error) { // Sanity check if blockNum < fromBlock && blockNum > toBlock { t.Fatal("invalid block number requested") } - return []std.Tx{exampleTx}, nil // 1 tx per block + return &client.Block{ + Height: blockNum, + Timestamp: blockTime.Add(time.Duration(blockNum) * time.Minute).UnixMilli(), + Txs: []std.Tx{exampleTx}, + }, nil // 1 tx per block }, } ) @@ -152,6 +160,7 @@ func TestBackup_ExecuteBackup_FixedRange(t *testing.T) { assert.Equal(t, expectedBlock, txData.BlockNum) assert.Equal(t, exampleTx, txData.Tx) + assert.Equal(t, blockTime.Add(time.Duration(expectedBlock)*time.Minute).Local(), time.UnixMilli(txData.Timestamp)) expectedBlock++ } @@ -183,11 +192,13 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) { cfg = DefaultConfig() + blockTime = time.Date(1987, 0o6, 24, 6, 32, 11, 0, time.FixedZone("Europe/Madrid", 0)) + mockClient = &mockClient{ getLatestBlockNumberFn: func() (uint64, error) { return toBlock, nil }, - getBlockTransactionsFn: func(blockNum uint64) ([]std.Tx, error) { + getBlockFn: func(blockNum uint64) (*client.Block, error) { // Sanity check if blockNum < fromBlock && blockNum > toBlock { t.Fatal("invalid block number requested") @@ -198,7 +209,11 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) { cancelFn() } - return []std.Tx{exampleTx}, nil // 1 tx per block + return &client.Block{ + Height: blockNum, + Timestamp: blockTime.Add(time.Duration(blockNum) * time.Minute).UnixMilli(), + Txs: []std.Tx{exampleTx}, + }, nil // 1 tx per block }, } ) @@ -246,6 +261,7 @@ func TestBackup_ExecuteBackup_Watch(t *testing.T) { assert.Equal(t, expectedBlock, txData.BlockNum) assert.Equal(t, exampleTx, txData.Tx) + assert.Equal(t, blockTime.Add(time.Duration(expectedBlock)*time.Minute).Local(), time.UnixMilli(txData.Timestamp)) expectedBlock++ } diff --git a/backup/client/client.go b/backup/client/client.go index 0755924..6abe7b9 100644 --- a/backup/client/client.go +++ b/backup/client/client.go @@ -1,13 +1,22 @@ package client -import "github.com/gnolang/gno/tm2/pkg/std" +import ( + "github.com/gnolang/gno/tm2/pkg/std" +) // Client defines the client interface for fetching chain data type Client interface { // GetLatestBlockNumber returns the latest block height from the chain GetLatestBlockNumber() (uint64, error) - // GetBlockTransactions returns the transactions contained - // within the specified block, if any - GetBlockTransactions(uint64) ([]std.Tx, error) + // GetBlock returns the transactions contained + // within the specified block, if any, apart from the block height and + // its timestamp in milliseconds. + GetBlock(uint64) (*Block, error) +} + +type Block struct { + Txs []std.Tx + Height uint64 + Timestamp int64 } diff --git a/backup/client/http/http.go b/backup/client/http/http.go index 77cfa51..a6963ab 100644 --- a/backup/client/http/http.go +++ b/backup/client/http/http.go @@ -4,13 +4,16 @@ package http import ( "fmt" + _ "github.com/gnolang/gno/gno.land/pkg/sdk/vm" "github.com/gnolang/gno/tm2/pkg/amino" rpcClient "github.com/gnolang/gno/tm2/pkg/bft/rpc/client" "github.com/gnolang/gno/tm2/pkg/std" - _ "github.com/gnolang/gno/gno.land/pkg/sdk/vm" + "github.com/gnolang/tx-archive/backup/client" ) +var _ client.Client = &Client{} + // Client is the TM2 HTTP client type Client struct { client rpcClient.Client @@ -40,7 +43,7 @@ func (c *Client) GetLatestBlockNumber() (uint64, error) { return uint64(status.SyncInfo.LatestBlockHeight), nil } -func (c *Client) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) { +func (c *Client) GetBlock(blockNum uint64) (*client.Block, error) { // Fetch the block blockNumInt64 := int64(blockNum) @@ -68,5 +71,9 @@ func (c *Client) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) { txs = append(txs, tx) } - return txs, nil + return &client.Block{ + Timestamp: block.Block.Time.UnixMilli(), + Height: blockNum, + Txs: txs, + }, nil } diff --git a/backup/mock_test.go b/backup/mock_test.go index 25fda87..a819843 100644 --- a/backup/mock_test.go +++ b/backup/mock_test.go @@ -1,15 +1,17 @@ package backup -import "github.com/gnolang/gno/tm2/pkg/std" +import ( + "github.com/gnolang/tx-archive/backup/client" +) type ( getLatestBlockNumberDelegate func() (uint64, error) - getBlockTransactionsDelegate func(uint64) ([]std.Tx, error) + getBlockDelegate func(uint64) (*client.Block, error) ) type mockClient struct { getLatestBlockNumberFn getLatestBlockNumberDelegate - getBlockTransactionsFn getBlockTransactionsDelegate + getBlockFn getBlockDelegate } func (m *mockClient) GetLatestBlockNumber() (uint64, error) { @@ -20,9 +22,9 @@ func (m *mockClient) GetLatestBlockNumber() (uint64, error) { return 0, nil } -func (m *mockClient) GetBlockTransactions(blockNum uint64) ([]std.Tx, error) { - if m.getBlockTransactionsFn != nil { - return m.getBlockTransactionsFn(blockNum) +func (m *mockClient) GetBlock(blockNum uint64) (*client.Block, error) { + if m.getBlockFn != nil { + return m.getBlockFn(blockNum) } return nil, nil diff --git a/restore/restore_test.go b/restore/restore_test.go index ec84276..2b0a710 100644 --- a/restore/restore_test.go +++ b/restore/restore_test.go @@ -7,9 +7,14 @@ import ( "testing" "time" + _ "github.com/gnolang/gno/gno.land/pkg/sdk/vm" // this is needed to load amino types + "github.com/gnolang/gno/tm2/pkg/amino" "github.com/gnolang/gno/tm2/pkg/std" - "github.com/gnolang/tx-archive/log/noop" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/gnolang/tx-archive/log/noop" + "github.com/gnolang/tx-archive/types" ) func TestRestore_ExecuteRestore(t *testing.T) { @@ -145,3 +150,37 @@ func TestRestore_ExecuteRestore_Watch(t *testing.T) { assert.Equal(t, exampleTx, tx) } } + +func TestRestore_BackwardCompatible(t *testing.T) { + t.Parallel() + + oldTx := `{"tx":{"msg":[{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9", + "send":"","pkg_path":"gno.land/r/demo/wugnot","func":"Approve","args": + ["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]},{"@type":"/vm.m_call","caller": + "g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/gns","func": + "Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8","18446744073709551615"]}, + {"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path": + "gno.land/r/demo/wugnot","func":"Approve","args":["g14fclvfqynndp0l6kpyxkpgn4sljw9rr96hz46l", + "18446744073709551615"]},{"@type":"/vm.m_call","caller": + "g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path":"gno.land/r/gnoswap/v2/position", + "func":"CollectFee","args":["26"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9", + "send":"","pkg_path":"gno.land/r/gnoswap/v2/staker","func":"CollectReward","args":["26","true"]}, + {"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"","pkg_path": + "gno.land/r/demo/wugnot","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8", + "18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9", + "send":"","pkg_path":"gno.land/r/gnoswap/v2/gns","func":"Approve","args":["g126swhfaq2vyvvjywevhgw7lv9hg8qan93dasu8", + "18446744073709551615"]},{"@type":"/vm.m_call","caller":"g1ngywvql2ql7t8uzl63w60eqcejkwg4rm4lxdw9","send":"", + "pkg_path":"gno.land/r/gnoswap/v2/position","func":"CollectFee","args":["146"]}],"fee":{"gas_wanted":"100000000", + "gas_fee":"1ugnot"},"signatures":[{"pub_key":{"@type":"/tm.PubKeySecp256k1", + "value":"Atgv/+TCwlR+jzjx94p4Ik0IuGET4J/q2q9ciaL4UOQh"}, + "signature":"iVfxsF37nRtgqyq9tMRMhyFLxp5RVdpI1r0mSHLmdg5aly0w82/in0ECey2PSpRk2UQ/fCtMpyOzaqIXiVKC4Q=="}], + "memo":""},"blockNum":"1194460"}` + + var out types.TxData + err := amino.UnmarshalJSON([]byte(oldTx), &out) + require.NoError(t, err) + + require.Zero(t, out.Timestamp) + require.Equal(t, uint64(0x1239dc), out.BlockNum) + require.Len(t, out.Tx.Msgs, 8) +} diff --git a/types/types.go b/types/types.go index e0be7cb..022d616 100644 --- a/types/types.go +++ b/types/types.go @@ -1,10 +1,15 @@ package types -import "github.com/gnolang/gno/tm2/pkg/std" +import ( + "github.com/gnolang/gno/tm2/pkg/std" +) // TxData contains the single block transaction, // along with the block information type TxData struct { Tx std.Tx `json:"tx"` BlockNum uint64 `json:"blockNum"` + + // Timestamp contains the block creation time in unix milliseconds + Timestamp int64 `json:"bt"` //nolint:tagliatelle // this name reduces disk space usage }