From c902e89763c18d23150357a87d892fbcc52c2f8b Mon Sep 17 00:00:00 2001 From: Milos Zivkovic Date: Wed, 4 Oct 2023 12:34:09 +0200 Subject: [PATCH] Streamline the tx data output --- backup/backup.go | 127 ++++++++++++------------------------------ backup/backup_test.go | 38 ++++++++----- types/types.go | 21 ++----- 3 files changed, 65 insertions(+), 121 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index 3ba27b3..4f6e50f 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -2,9 +2,8 @@ package backup import ( "encoding/json" - "errors" "fmt" - "os" + "io" "github.com/gnolang/tx-archive/backup/client" "github.com/gnolang/tx-archive/log" @@ -14,6 +13,7 @@ import ( // ExecuteBackup executes the node backup process func ExecuteBackup( client client.Client, + writer io.Writer, logger log.Logger, cfg Config, ) error { @@ -22,37 +22,6 @@ func ExecuteBackup( return fmt.Errorf("invalid config, %w", cfgErr) } - // Open the file for writing - outputFile, openErr := os.OpenFile( - cfg.OutputFile, - os.O_RDWR|os.O_CREATE|os.O_TRUNC, - 0o755, - ) - if openErr != nil { - return fmt.Errorf("unable to open file %s, %w", cfg.OutputFile, openErr) - } - - closeFile := func() error { - if err := outputFile.Close(); err != nil { - logger.Error("unable to close output file", "err", err.Error()) - - return err - } - - return nil - } - - teardown := func() { - if err := closeFile(); err != nil { - if removeErr := os.Remove(outputFile.Name()); removeErr != nil { - logger.Error("unable to remove file", "err", err.Error()) - } - } - } - - // Set up the teardown - defer teardown() - // Determine the right bound toBlock, boundErr := determineRightBound(client, cfg.ToBlock) if boundErr != nil { @@ -60,32 +29,27 @@ func ExecuteBackup( } // Gather the chain data from the node - blockData, blockDataErr := getBlockData(client, logger, cfg.FromBlock, toBlock) - if blockDataErr != nil { - return fmt.Errorf("unable to fetch block data, %w", blockDataErr) - } - - // Prepare the archive - metadata, metadataErr := generateMetadata(blockData) - if metadataErr != nil { - return fmt.Errorf("unable to generate metadata, %w", metadataErr) - } + for block := cfg.FromBlock; block <= toBlock; block++ { + txs, txErr := client.GetBlockTransactions(block) + if txErr != nil { + return fmt.Errorf("unable to fetch block transactions, %w", txErr) + } - archive := &types.Archive{ - BlockData: blockData, - Metadata: metadata, - } + // Save the block transaction data, if any + for _, tx := range txs { + data := &types.TxData{ + Tx: tx, + BlockNum: block, + } - // Marshal the archive data - archiveRaw, marshalErr := json.Marshal(archive) - if marshalErr != nil { - return fmt.Errorf("unable to marshal archive JSON, %w", marshalErr) - } + // Write the tx data to the file + if writeErr := writeTxData(writer, data); writeErr != nil { + return fmt.Errorf("unable to write tx data, %w", writeErr) + } + } - // Write the archive data to a file - _, writeErr := outputFile.Write(archiveRaw) - if writeErr != nil { - return fmt.Errorf("unable to write archive JSON, %w", writeErr) + // Log the progress + logProgress(logger, cfg.FromBlock, toBlock, block) } return nil @@ -113,33 +77,27 @@ func determineRightBound( return latestBlockNumber, nil } -// getBlockData fetches the block data from the chain -func getBlockData( - client client.Client, - logger log.Logger, - from, - to uint64, -) ([]*types.BlockData, error) { - blockData := make([]*types.BlockData, 0, to-from+1) - - for block := from; block <= to; block++ { - txs, txErr := client.GetBlockTransactions(block) - if txErr != nil { - return nil, fmt.Errorf("unable to fetch block transactions, %w", txErr) - } +// writeTxData outputs the tx data to the writer +func writeTxData(writer io.Writer, txData *types.TxData) error { + // Marshal tx data into JSON + jsonData, err := json.Marshal(txData) + if err != nil { + return fmt.Errorf("unable to marshal JSON data, %w", err) + } - // Save the block transaction data - data := &types.BlockData{ - Txs: txs, - BlockNum: block, - } - blockData = append(blockData, data) + // Write the JSON data as a line to the file + _, err = writer.Write(jsonData) + if err != nil { + return fmt.Errorf("unable to write to output, %w", err) + } - // Log the progress - logProgress(logger, from, to, block) + // Write a newline character to separate JSON objects + _, err = writer.Write([]byte("\n")) + if err != nil { + return fmt.Errorf("unable to write newline output, %w", err) } - return blockData, nil + return nil } // logProgress logs the backup progress @@ -155,14 +113,3 @@ func logProgress(logger log.Logger, from, to, current uint64) { "status", fmt.Sprintf("%.2f%%", status), ) } - -func generateMetadata(blockData []*types.BlockData) (*types.Metadata, error) { - if len(blockData) == 0 { - return nil, errors.New("unable to generate metadata, no block data") - } - - return &types.Metadata{ - EarliestBlockHeight: blockData[0].BlockNum, - LatestBlockHeight: blockData[len(blockData)-1].BlockNum, - }, nil -} diff --git a/backup/backup_test.go b/backup/backup_test.go index ac0d8eb..fdbaf54 100644 --- a/backup/backup_test.go +++ b/backup/backup_test.go @@ -1,6 +1,7 @@ package backup import ( + "bufio" "encoding/json" "errors" "os" @@ -102,7 +103,7 @@ func TestBackup_ExecuteBackup(t *testing.T) { t.Fatal("invalid block number requested") } - return []std.Tx{exampleTx}, nil + return []std.Tx{exampleTx}, nil // 1 tx per block }, } ) @@ -120,27 +121,34 @@ func TestBackup_ExecuteBackup(t *testing.T) { cfg.Overwrite = true // Run the backup procedure - require.NoError(t, ExecuteBackup(mockClient, noop.New(), cfg)) + require.NoError(t, ExecuteBackup(mockClient, tempFile, noop.New(), cfg)) // Read the output file - archiveRaw, err := os.ReadFile(tempFile.Name()) + fileRaw, err := os.Open(tempFile.Name()) require.NoError(t, err) - // Unmarshal the raw archive output - var archive types.Archive + // Set up a line-by-line scanner + scanner := bufio.NewScanner(fileRaw) - require.NoError(t, json.Unmarshal(archiveRaw, &archive)) + expectedBlock := fromBlock - // Validate the archive - assert.Equal(t, fromBlock, archive.Metadata.EarliestBlockHeight) - assert.Equal(t, toBlock, archive.Metadata.LatestBlockHeight) - assert.Equal(t, int(toBlock-fromBlock+1), len(archive.BlockData)) + // Iterate over each line in the file + for scanner.Scan() { + var txData types.TxData - for index, block := range archive.BlockData { - assert.Equal(t, uint64(index)+fromBlock, block.BlockNum) - - for _, tx := range block.Txs { - assert.Equal(t, exampleTx, tx) + // Unmarshal the JSON data into the Person struct + if err := json.Unmarshal(scanner.Bytes(), &txData); err != nil { + t.Fatalf("unable to unmarshal JSON line, %v", err) } + + assert.Equal(t, expectedBlock, txData.BlockNum) + assert.Equal(t, exampleTx, txData.Tx) + + expectedBlock++ + } + + // Check for errors during scanning + if err := scanner.Err(); err != nil { + t.Fatalf("error encountered during scan, %v", err) } } diff --git a/types/types.go b/types/types.go index d5409ff..e0be7cb 100644 --- a/types/types.go +++ b/types/types.go @@ -2,20 +2,9 @@ package types import "github.com/gnolang/gno/tm2/pkg/std" -// Archive wraps the backed-up chain data -type Archive struct { - Metadata *Metadata `json:"metadata"` - BlockData []*BlockData `json:"blockData"` -} - -// Metadata contains contextual information about the archive -type Metadata struct { - EarliestBlockHeight uint64 `json:"earliestBlockHeight"` - LatestBlockHeight uint64 `json:"latestBlockHeight"` -} - -// BlockData contains the historical transaction data -type BlockData struct { - Txs []std.Tx `json:"txs"` - BlockNum uint64 `json:"blockNum"` +// TxData contains the single block transaction, +// along with the block information +type TxData struct { + Tx std.Tx `json:"tx"` + BlockNum uint64 `json:"blockNum"` }