Streamline the tx data output

backup/backup.go

@@ -2,9 +2,8 @@ package backup
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
-	"os"
+	"io"
 
 	"github.com/gnolang/tx-archive/backup/client"
 	"github.com/gnolang/tx-archive/log"
@@ -14,6 +13,7 @@ import (
 // ExecuteBackup executes the node backup process
 func ExecuteBackup(
 	client client.Client,
+	writer io.Writer,
 	logger log.Logger,
 	cfg Config,
 ) error {
@@ -22,70 +22,34 @@ func ExecuteBackup(
 		return fmt.Errorf("invalid config, %w", cfgErr)
 	}
 
-	// Open the file for writing
-	outputFile, openErr := os.OpenFile(
-		cfg.OutputFile,
-		os.O_RDWR|os.O_CREATE|os.O_TRUNC,
-		0o755,
-	)
-	if openErr != nil {
-		return fmt.Errorf("unable to open file %s, %w", cfg.OutputFile, openErr)
-	}
-
-	closeFile := func() error {
-		if err := outputFile.Close(); err != nil {
-			logger.Error("unable to close output file", "err", err.Error())
-
-			return err
-		}
-
-		return nil
-	}
-
-	teardown := func() {
-		if err := closeFile(); err != nil {
-			if removeErr := os.Remove(outputFile.Name()); removeErr != nil {
-				logger.Error("unable to remove file", "err", err.Error())
-			}
-		}
-	}
-
-	// Set up the teardown
-	defer teardown()
-
 	// Determine the right bound
 	toBlock, boundErr := determineRightBound(client, cfg.ToBlock)
 	if boundErr != nil {
 		return fmt.Errorf("unable to determine right bound, %w", boundErr)
 	}
 
 	// Gather the chain data from the node
-	blockData, blockDataErr := getBlockData(client, logger, cfg.FromBlock, toBlock)
-	if blockDataErr != nil {
-		return fmt.Errorf("unable to fetch block data, %w", blockDataErr)
-	}
-
-	// Prepare the archive
-	metadata, metadataErr := generateMetadata(blockData)
-	if metadataErr != nil {
-		return fmt.Errorf("unable to generate metadata, %w", metadataErr)
-	}
+	for block := cfg.FromBlock; block <= toBlock; block++ {
+		txs, txErr := client.GetBlockTransactions(block)
+		if txErr != nil {
+			return fmt.Errorf("unable to fetch block transactions, %w", txErr)
+		}
 
-	archive := &types.Archive{
-		BlockData: blockData,
-		Metadata:  metadata,
-	}
+		// Save the block transaction data, if any
+		for _, tx := range txs {
+			data := &types.TxData{
+				Tx:       tx,
+				BlockNum: block,
+			}
 
-	// Marshal the archive data
-	archiveRaw, marshalErr := json.Marshal(archive)
-	if marshalErr != nil {
-		return fmt.Errorf("unable to marshal archive JSON, %w", marshalErr)
-	}
+			// Write the tx data to the file
+			if writeErr := writeTxData(writer, data); writeErr != nil {
+				return fmt.Errorf("unable to write tx data, %w", writeErr)
+			}
+		}
 
-	// Write the archive data to a file
-	_, writeErr := outputFile.Write(archiveRaw)
-	if writeErr != nil {
-		return fmt.Errorf("unable to write archive JSON, %w", writeErr)
+		// Log the progress
+		logProgress(logger, cfg.FromBlock, toBlock, block)
 	}
 
 	return nil
@@ -113,33 +77,27 @@ func determineRightBound(
 	return latestBlockNumber, nil
 }
 
-// getBlockData fetches the block data from the chain
-func getBlockData(
-	client client.Client,
-	logger log.Logger,
-	from,
-	to uint64,
-) ([]*types.BlockData, error) {
-	blockData := make([]*types.BlockData, 0, to-from+1)
-
-	for block := from; block <= to; block++ {
-		txs, txErr := client.GetBlockTransactions(block)
-		if txErr != nil {
-			return nil, fmt.Errorf("unable to fetch block transactions, %w", txErr)
-		}
+// writeTxData outputs the tx data to the writer
+func writeTxData(writer io.Writer, txData *types.TxData) error {
+	// Marshal tx data into JSON
+	jsonData, err := json.Marshal(txData)
+	if err != nil {
+		return fmt.Errorf("unable to marshal JSON data, %w", err)
+	}
 
-		// Save the block transaction data
-		data := &types.BlockData{
-			Txs:      txs,
-			BlockNum: block,
-		}
-		blockData = append(blockData, data)
+	// Write the JSON data as a line to the file
+	_, err = writer.Write(jsonData)
+	if err != nil {
+		return fmt.Errorf("unable to write to output, %w", err)
+	}
 
-		// Log the progress
-		logProgress(logger, from, to, block)
+	// Write a newline character to separate JSON objects
+	_, err = writer.Write([]byte("\n"))
+	if err != nil {
+		return fmt.Errorf("unable to write newline output, %w", err)
 	}
 
-	return blockData, nil
+	return nil
 }
 
 // logProgress logs the backup progress
@@ -155,14 +113,3 @@ func logProgress(logger log.Logger, from, to, current uint64) {
 		"status", fmt.Sprintf("%.2f%%", status),
 	)
 }
-
-func generateMetadata(blockData []*types.BlockData) (*types.Metadata, error) {
-	if len(blockData) == 0 {
-		return nil, errors.New("unable to generate metadata, no block data")
-	}
-
-	return &types.Metadata{
-		EarliestBlockHeight: blockData[0].BlockNum,
-		LatestBlockHeight:   blockData[len(blockData)-1].BlockNum,
-	}, nil
-}

backup/backup_test.go

@@ -1,6 +1,7 @@
 package backup
 
 import (
+	"bufio"
 	"encoding/json"
 	"errors"
 	"os"
@@ -102,7 +103,7 @@ func TestBackup_ExecuteBackup(t *testing.T) {
 					t.Fatal("invalid block number requested")
 				}
 
-				return []std.Tx{exampleTx}, nil
+				return []std.Tx{exampleTx}, nil // 1 tx per block
 			},
 		}
 	)
@@ -120,27 +121,34 @@ func TestBackup_ExecuteBackup(t *testing.T) {
 	cfg.Overwrite = true
 
 	// Run the backup procedure
-	require.NoError(t, ExecuteBackup(mockClient, noop.New(), cfg))
+	require.NoError(t, ExecuteBackup(mockClient, tempFile, noop.New(), cfg))
 
 	// Read the output file
-	archiveRaw, err := os.ReadFile(tempFile.Name())
+	fileRaw, err := os.Open(tempFile.Name())
 	require.NoError(t, err)
 
-	// Unmarshal the raw archive output
-	var archive types.Archive
+	// Set up a line-by-line scanner
+	scanner := bufio.NewScanner(fileRaw)
 
-	require.NoError(t, json.Unmarshal(archiveRaw, &archive))
+	expectedBlock := fromBlock
 
-	// Validate the archive
-	assert.Equal(t, fromBlock, archive.Metadata.EarliestBlockHeight)
-	assert.Equal(t, toBlock, archive.Metadata.LatestBlockHeight)
-	assert.Equal(t, int(toBlock-fromBlock+1), len(archive.BlockData))
+	// Iterate over each line in the file
+	for scanner.Scan() {
+		var txData types.TxData
 
-	for index, block := range archive.BlockData {
-		assert.Equal(t, uint64(index)+fromBlock, block.BlockNum)
-
-		for _, tx := range block.Txs {
-			assert.Equal(t, exampleTx, tx)
+		// Unmarshal the JSON data into the Person struct
+		if err := json.Unmarshal(scanner.Bytes(), &txData); err != nil {
+			t.Fatalf("unable to unmarshal JSON line, %v", err)
 		}
+
+		assert.Equal(t, expectedBlock, txData.BlockNum)
+		assert.Equal(t, exampleTx, txData.Tx)
+
+		expectedBlock++
+	}
+
+	// Check for errors during scanning
+	if err := scanner.Err(); err != nil {
+		t.Fatalf("error encountered during scan, %v", err)
 	}
 }

types/types.go

@@ -2,20 +2,9 @@ package types
 
 import "github.com/gnolang/gno/tm2/pkg/std"
 
-// Archive wraps the backed-up chain data
-type Archive struct {
-	Metadata  *Metadata    `json:"metadata"`
-	BlockData []*BlockData `json:"blockData"`
-}
-
-// Metadata contains contextual information about the archive
-type Metadata struct {
-	EarliestBlockHeight uint64 `json:"earliestBlockHeight"`
-	LatestBlockHeight   uint64 `json:"latestBlockHeight"`
-}
-
-// BlockData contains the historical transaction data
-type BlockData struct {
-	Txs      []std.Tx `json:"txs"`
-	BlockNum uint64   `json:"blockNum"`
+// TxData contains the single block transaction,
+// along with the block information
+type TxData struct {
+	Tx       std.Tx `json:"tx"`
+	BlockNum uint64 `json:"blockNum"`
 }