diff --git a/cmd/check_construction.go b/cmd/check_construction.go index c05c6745..3371878e 100644 --- a/cmd/check_construction.go +++ b/cmd/check_construction.go @@ -149,6 +149,7 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) g.Go(func() error { return constructionTester.StartPeriodicLogger(ctx) diff --git a/cmd/check_data.go b/cmd/check_data.go index 81637654..5d338b40 100644 --- a/cmd/check_data.go +++ b/cmd/check_data.go @@ -156,6 +156,7 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error { g, ctx := errgroup.WithContext(ctx) ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID) + ctx = logger.AddInfoMetaDataToContext(ctx, Config.InfoMetaData) g.Go(func() error { return dataTester.StartPeriodicLogger(ctx) diff --git a/cmd/root.go b/cmd/root.go index 6e50a0db..747239ac 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,6 +59,7 @@ var ( tableSize int64 requestUUID string statusPort uint + InfoMetaData string // Config is the populated *configuration.Configuration from // the configurationFile. If none is provided, this is set @@ -289,6 +290,13 @@ default values.`, "requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID", ) + checkDataCmd.Flags().StringVar( + &InfoMetaData, + "info-metadata", + "", + "metadata configures the metadata which aims to show in logs", + ) + checkDataCmd.Flags().UintVar( &statusPort, "status-port", @@ -441,6 +449,10 @@ func initConfig() { Config.Construction.StatusPort = statusPort } } + + if len(InfoMetaData) != 0 { + Config.InfoMetaData = InfoMetaData + } } func ensureDataDirectoryExists() { diff --git a/configuration/types.go b/configuration/types.go index b7b5d7bc..617d7100 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -461,6 +461,10 @@ type Configuration struct { // then this value must be true. CoinSupported bool `json:"coin_supported"` + // InfoMetaData is a map of key:value + // which aims to show in the log for search + InfoMetaData string `json:"info_metadata,omitempty"` + Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` Perf *CheckPerfConfiguration `json:"perf"` diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 10ea9159..427d9a38 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -20,6 +20,7 @@ import ( "log" "os" "path" + "strings" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -43,6 +44,8 @@ type contextKey int const ( RequestUUID contextKey = iota + InfoMetaDataKey contextKey = iota + // blockStreamFile contains the stream of processed // blocks and whether they were added or removed. blockStreamFile = "blocks.txt" @@ -83,6 +86,7 @@ type Logger struct { logBalanceChanges bool logReconciliation bool logRequestUUID string + logInfoMetaData map[string]string lastStatsMessage string lastProgressMessage string @@ -100,6 +104,7 @@ func NewLogger( checkType CheckType, network *types.NetworkIdentifier, logRequestUUID string, + logInfoMetaData map[string]string, fields ...zap.Field, ) (*Logger, error) { zapLogger, err := buildZapLogger(checkType, network, logRequestUUID, fields...) @@ -113,6 +118,7 @@ func NewLogger( logBalanceChanges: logBalanceChanges, logReconciliation: logReconciliation, logRequestUUID: logRequestUUID, + logInfoMetaData: logInfoMetaData, zapLogger: zapLogger, }, nil } @@ -161,6 +167,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta ) statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) + statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) // Don't print out the same stats message twice. if statsMessage == l.lastStatsMessage { @@ -187,6 +194,7 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta ) progressMessage = AddRequestUUID(progressMessage, l.logRequestUUID) + progressMessage = AddInfoMetaData(progressMessage, l.logInfoMetaData) // Don't print out the same progress message twice. if progressMessage == l.lastProgressMessage { @@ -216,7 +224,8 @@ func (l *Logger) LogConstructionStatus( } statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID) - + statsMessage = AddInfoMetaData(statsMessage, l.logInfoMetaData) + l.lastStatsMessage = statsMessage color.Cyan(statsMessage) } @@ -232,6 +241,7 @@ func LogMemoryStats(ctx context.Context) { memUsage.GarbageCollections, ) statsMessage = AddRequestUUIDFromContext(ctx, statsMessage) + statsMessage = AddInfoMetaDataFromContext(ctx, statsMessage) color.Cyan(statsMessage) } @@ -267,6 +277,7 @@ func (l *Logger) AddBlockStream( block.ParentBlockIdentifier.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) + blockString = AddInfoMetaData(blockString, l.logInfoMetaData) color.Cyan(blockString) if _, err := f.WriteString(blockString); err != nil { return fmt.Errorf("failed to write block string %s: %w", blockString, err) @@ -305,6 +316,7 @@ func (l *Logger) RemoveBlockStream( block.Hash, ) blockString = AddRequestUUID(blockString, l.logRequestUUID) + blockString = AddInfoMetaData(blockString, l.logInfoMetaData) color.Cyan(blockString) _, err = f.WriteString(blockString) if err != nil { @@ -347,6 +359,7 @@ func (l *Logger) TransactionStream( block.BlockIdentifier.Hash, ) transactionString = AddRequestUUID(transactionString, l.logRequestUUID) + transactionString = AddInfoMetaData(transactionString, l.logInfoMetaData) color.Cyan(transactionString) _, err = f.WriteString(transactionString) if err != nil { @@ -383,6 +396,7 @@ func (l *Logger) TransactionStream( *op.Status, ) transactionOperationString = AddRequestUUID(transactionOperationString, l.logRequestUUID) + transactionOperationString = AddInfoMetaData(transactionOperationString, l.logInfoMetaData) color.Cyan(transactionOperationString) _, err = f.WriteString(transactionOperationString) if err != nil { @@ -429,6 +443,7 @@ func (l *Logger) BalanceStream( balanceChange.Block.Hash, ) balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID) + balanceLog = AddInfoMetaData(balanceLog, l.logInfoMetaData) color.Cyan(balanceLog) if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil { err = fmt.Errorf("failed to write balance log %s: %w", balanceLog, err) @@ -473,6 +488,7 @@ func (l *Logger) ReconcileSuccessStream( block.Index, ) reconciledLog = AddRequestUUID(reconciledLog, l.logRequestUUID) + reconciledLog = AddInfoMetaData(reconciledLog, l.logInfoMetaData) color.Cyan(reconciledLog) reconciliationSuccessString := fmt.Sprintf( @@ -485,6 +501,7 @@ func (l *Logger) ReconcileSuccessStream( block.Hash, ) reconciliationSuccessString = AddRequestUUID(reconciliationSuccessString, l.logRequestUUID) + reconciliationSuccessString = AddInfoMetaData(reconciliationSuccessString, l.logInfoMetaData) color.Cyan(reconciliationSuccessString) _, err = f.WriteString(reconciliationSuccessString) @@ -558,6 +575,7 @@ func (l *Logger) ReconcileFailureStream( liveBalance, ) reconciliationFailureString = AddRequestUUID(reconciliationFailureString, l.logRequestUUID) + reconciliationFailureString = AddInfoMetaData(reconciliationFailureString, l.logInfoMetaData) color.Cyan(reconciliationFailureString) _, err = f.WriteString(reconciliationFailureString) if err != nil { @@ -649,3 +667,60 @@ func requestUUIDFromContext(ctx context.Context) string { return "" } } + +// Add InfoMetaData k-v pairs to the tip +func AddInfoMetaDataFromContext(ctx context.Context, msg string) string { + logInfoMetaData := InfoMetaDataFromContext(ctx) + if len(logInfoMetaData) != 0 { + for k, v := range logInfoMetaData { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } + } + return msg +} + +// Add InfoMetaData k-v pairs to the tip +func AddInfoMetaData(msg string, logInfoMetaData map[string]string) string { + if len(logInfoMetaData) != 0 { + for k, v := range logInfoMetaData { + msg = fmt.Sprintf("%s, %s: %s", msg, k, v) + } + } + return msg +} + +// AddInfoMetaDataToContext will add InfoMetaData to the context, and return the new context +func AddInfoMetaDataToContext(ctx context.Context, InfoMetaData string) context.Context { + return context.WithValue(ctx, InfoMetaDataKey, InfoMetaData) +} + +// InfoMetaDataFromContext is used to extract InfoMetaData from a context +func InfoMetaDataFromContext(ctx context.Context) map[string]string { + var metadata string + switch v := ctx.Value(InfoMetaDataKey).(type) { + case string: + metadata = v + default: + metadata = "" + } + return ConvertStringToMap(metadata) +} + +// ConvertStringToMap is used to convert a string to map by split , and ; +func ConvertStringToMap(metadata string) map[string]string { + InfoMetaDataMap := make(map[string]string) + if len(metadata) == 0 { + return InfoMetaDataMap + } + pairs := strings.Split(metadata, ",") + for _, pair := range pairs { + kv := strings.Split(pair, ":") + if(len(kv) != 2) { + log := fmt.Sprintf("the %s from %s could be transfer to key value pair", pair, metadata) + color.Yellow(log) + } else { + InfoMetaDataMap[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1]) + } + } + return InfoMetaDataMap +} diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go index ba8a806f..ad26185c 100644 --- a/pkg/tester/construction.go +++ b/pkg/tester/construction.go @@ -115,6 +115,7 @@ func InitializeConstruction( } counterStorage := modules.NewCounterStorage(localStore) + logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) logger, err := logger.NewLogger( dataPath, false, @@ -124,6 +125,7 @@ func InitializeConstruction( logger.Construction, network, config.RequestUUID, + logInfoMetaData, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) diff --git a/pkg/tester/data.go b/pkg/tester/data.go index fbf4d189..b687e885 100644 --- a/pkg/tester/data.go +++ b/pkg/tester/data.go @@ -232,6 +232,7 @@ func InitializeData( blockStorage := modules.NewBlockStorage(localStore, config.SerialBlockWorkers) balanceStorage := modules.NewBalanceStorage(localStore) + logInfoMetaData := logger.ConvertStringToMap(config.InfoMetaData) logger, err := logger.NewLogger( dataPath, config.Data.LogBlocks, @@ -241,6 +242,7 @@ func InitializeData( logger.Data, network, config.RequestUUID, + logInfoMetaData, ) if err != nil { return nil, fmt.Errorf("unable to initialize logger with error: %w", err) @@ -1089,6 +1091,7 @@ func (t *DataTester) recursiveOpSearch( logger.Data, t.network, EmptyRequestUUID, + nil, ) if err != nil {