feat(log): move processing info log to debug level, add builder log

cmd/pimo/main.go

@@ -146,7 +146,6 @@ func run() {
 	pipeline := model.NewPipeline(source).
 		Process(model.NewCounterProcessWithCallback("input-line", 0, updateContext)).
 		Process(model.NewRepeaterProcess(iteration))
-	over.AddGlobalFields("input-line")
 
 	var (
 		err    error
@@ -190,6 +189,7 @@ func run() {
 	statistics.Reset()
 	startTime := time.Now()
 
+	over.AddGlobalFields("input-line")
 	over.AddGlobalFields("output-line")
 	err = pipeline.AddSink(jsonline.NewSinkWithContext(os.Stdout, "output-line")).Run()
 

pkg/add/add.go

@@ -18,6 +18,8 @@
 package add
 
 import (
+	"fmt"
+
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/rs/zerolog/log"
 )
@@ -34,7 +36,7 @@ func NewMask(value model.Entry) MaskEngine {
 
 // MaskContext add the field
 func (am MaskEngine) MaskContext(context model.Dictionary, key string, contexts ...model.Dictionary) (model.Dictionary, error) {
-	log.Info().Msg("Mask add")
+	log.Debug().Msg("Mask add")
 	_, present := context.GetValue(key)
 	if !present {
 		context.Set(key, am.value)
@@ -43,6 +45,10 @@ func (am MaskEngine) MaskContext(context model.Dictionary, key string, contexts
 	return context, nil
 }
 
+func (am MaskEngine) Name() string {
+	return fmt.Sprintf("add %v", am.value)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskContextEngine, bool, error) {
 	if conf.Mask.Add != nil {

pkg/command/command.go

@@ -18,6 +18,7 @@
 package command
 
 import (
+	"fmt"
 	"os/exec"
 	"strings"
 
@@ -37,7 +38,7 @@ func NewMask(cmd string) MaskEngine {
 
 // Mask delegate mask algorithm to an external program
 func (cme MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask command")
+	log.Debug().Msg("Mask command")
 	splitCommand := strings.Split(cme.Cmd, " ")
 	/* #nosec */
 	out, err := exec.Command(splitCommand[0], splitCommand[1:]...).Output()
@@ -49,6 +50,10 @@ func (cme MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.En
 	return resulting, nil
 }
 
+func (cme MaskEngine) Name() string {
+	return fmt.Sprintf("command %s", cme.Cmd)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if len(conf.Mask.Command) != 0 {

pkg/constant/constant.go

@@ -18,6 +18,8 @@
 package constant
 
 import (
+	"fmt"
+
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/rs/zerolog/log"
 )
@@ -34,10 +36,14 @@ func NewMask(data model.Entry) MaskEngine {
 
 // Mask return a Constant from a MaskEngine
 func (cm MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask constant")
+	log.Debug().Msg("Mask constant")
 	return cm.constValue, nil
 }
 
+func (cm MaskEngine) Name() string {
+	return fmt.Sprintf("constant %s", cm.constValue)
+}
+
 // Factory create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if conf.Mask.Constant != nil {

pkg/dateparser/dateparser.go

@@ -38,7 +38,7 @@ func NewMask(input, output string) MaskEngine {
 
 // Mask change a time format
 func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask dateparser")
+	log.Debug().Msg("Mask dateparser")
 	var t time.Time
 	var err error
 	if me.inputFormat != "" {
@@ -66,6 +66,10 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 	return t, nil
 }
 
+func (cme MaskEngine) Name() string {
+	return fmt.Sprintf("dateparser inputFormat='%s' outputFormat='%s'", cme.inputFormat, cme.outputFormat)
+}
+
 // Factory Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if len(conf.Mask.DateParser.InputFormat) != 0 || len(conf.Mask.DateParser.OutputFormat) != 0 {

pkg/duration/duration.go

@@ -77,7 +77,7 @@ func ParseInt64(value string) int64 {
 
 // Mask masks a time value with a duration
 func (dura MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask duration")
+	log.Debug().Msg("Mask duration")
 	var t time.Time
 	var err error
 	switch v := e.(type) {
@@ -94,6 +94,10 @@ func (dura MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.E
 	return t.Add(dura.duration), nil
 }
 
+func (dura MaskEngine) Name() string {
+	return fmt.Sprintf("duration %s", dura.duration)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if len(conf.Mask.Duration) != 0 {

pkg/ff1/ff1.go

@@ -48,7 +48,7 @@ func (ff1m MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.E
 		return e, nil
 	}
 
-	log.Info().Msg("Mask ff1")
+	log.Debug().Msg("Mask ff1")
 
 	// Extract tweak from the Dictionary (context)
 	var tweak string
@@ -95,6 +95,10 @@ func decodingKey(key string) ([]byte, error) {
 	return decodedkey, nil
 }
 
+func (ff1m MaskEngine) Name() string {
+	return fmt.Sprintf("dateparser keyFromEnv='%s' radix='%d' tweakField='%s' decrypt='%v'", ff1m.keyFromEnv, ff1m.radix, ff1m.tweakField, ff1m.decrypt)
+}
+
 // Factory create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if conf.Mask.FF1.KeyFromEnv != "" || conf.Mask.FF1.Radix > 0 {

pkg/fluxuri/fluxuri.go

@@ -18,6 +18,8 @@
 package fluxuri
 
 import (
+	"fmt"
+
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/cgi-fr/pimo/pkg/uri"
 	"github.com/rs/zerolog/log"
@@ -43,14 +45,18 @@ func NewMask(name string) (MaskEngine, error) {
 
 // MaskContext add the field if not existing or replace the value if existing
 func (me MaskEngine) MaskContext(context model.Dictionary, key string, contexts ...model.Dictionary) (model.Dictionary, error) {
-	log.Info().Msg("Mask fluxuri")
+	log.Debug().Msg("Mask fluxuri")
 	if *me.Actual < me.LenList {
 		context.Set(key, me.List[*me.Actual])
 		*me.Actual++
 	}
 	return context, nil
 }
 
+func (me MaskEngine) Name() string {
+	return fmt.Sprintf("fluxuri size=%d", len(me.List))
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskContextEngine, bool, error) {
 	if len(conf.Mask.FluxURI) != 0 {

pkg/hash/hash.go

@@ -33,12 +33,16 @@ type MaskEngine struct {
 
 // Mask choose a mask value by hash
 func (hm MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask hash")
+	log.Debug().Msg("Mask hash")
 	h := fnv.New32a()
 	_, err := h.Write([]byte(e.(string)))
 	return hm.List[int(h.Sum32())%len(hm.List)], err
 }
 
+func (hm MaskEngine) Name() string {
+	return fmt.Sprintf("hash size=%d", len(hm.List))
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if len(conf.Mask.Hash) != 0 && len(conf.Mask.HashInURI) != 0 {

pkg/increment/increment.go

@@ -18,6 +18,8 @@
 package increment
 
 import (
+	"fmt"
+
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/rs/zerolog/log"
 )
@@ -36,12 +38,16 @@ func NewMask(start, incr int) MaskEngine {
 
 // Mask masks a value with an incremental int
 func (incr MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask increment")
+	log.Debug().Msg("Mask increment")
 	output := *incr.Value
 	*incr.Value += incr.Increment
 	return output, nil
 }
 
+func (incr MaskEngine) Name() string {
+	return fmt.Sprintf("increment increment=%d start=%d", incr.Increment, incr.Value)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if conf.Mask.Incremental.Increment != 0 {

pkg/model/cache.go

@@ -126,6 +126,10 @@ func (mce MaskCacheEngine) Mask(e Entry, context ...Dictionary) (Entry, error) {
 	return value, err
 }
 
+func (mce MaskCacheEngine) Name() string {
+	return fmt.Sprintf("cached mask of %s", mce.OriginalEngine.Name())
+}
+
 type UniqueMaskCacheEngine struct {
 	cache          UniqueCache
 	originalEngine MaskEngine
@@ -156,6 +160,10 @@ func (umce UniqueMaskCacheEngine) Mask(e Entry, context ...Dictionary) (Entry, e
 	return nil, fmt.Errorf("Unique value not found")
 }
 
+func (umce UniqueMaskCacheEngine) Name() string {
+	return fmt.Sprintf("unique cache of %s", umce.originalEngine.Name())
+}
+
 func NewFromCacheProcess(selector Selector, cache Cache) Processor {
 	return &FromCacheProcess{selector, cache, &QueueCollector{}, map[Entry]*QueueCollector{}}
 }

pkg/model/model.go

@@ -24,11 +24,13 @@ import (
 // MaskEngine is a masking algorithm
 type MaskEngine interface {
 	Mask(Entry, ...Dictionary) (Entry, error)
+	Name() string
 }
 
 // MaskContextEngine is a masking algorithm for dictionary
 type MaskContextEngine interface {
 	MaskContext(Dictionary, string, ...Dictionary) (Dictionary, error)
+	Name() string
 }
 
 // FunctionMaskEngine implements MaskEngine with a simple function
@@ -41,6 +43,10 @@ func (fme FunctionMaskEngine) Mask(e Entry, context ...Dictionary) (Entry, error
 	return fme.Function(e, context...)
 }
 
+func (fme FunctionMaskEngine) Name() string {
+	return "function mask engine"
+}
+
 type MaskFactory func(Masking, int64, map[string]Cache) (MaskEngine, bool, error)
 
 type MaskContextFactory func(Masking, int64, map[string]Cache) (MaskContextEngine, bool, error)

pkg/model/model_test.go

@@ -235,6 +235,10 @@ func (am TestAddMaskEngine) MaskContext(context Dictionary, key string, contexts
 	return context, nil
 }
 
+func (am TestAddMaskEngine) Name() string {
+	return "test function"
+}
+
 func TestMaskEngineWithContext(t *testing.T) {
 	mySlice := []Dictionary{NewDictionary().With("city", "Nantes")}
 	var result []Dictionary

pkg/model/pipeline.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/goccy/go-yaml"
+	"github.com/rs/zerolog/log"
 )
 
 // nolint: gochecknoglobals
@@ -74,6 +75,7 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache)
 				return nil, nil, errors.New("Cache '" + v.Cache + "' not found for '" + v.Selector.Jsonpath + "'")
 			}
 			pipeline = pipeline.Process(NewFromCacheProcess(NewPathSelector(v.Selector.Jsonpath), cache))
+			log.Info().Str("path", v.Selector.Jsonpath).Interface("mask", v.Mask.FromCache).Msg("Add mask")
 			nbArg++
 		}
 
@@ -96,6 +98,7 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache)
 					}
 				}
 				pipeline = pipeline.Process(NewMaskEngineProcess(NewPathSelector(v.Selector.Jsonpath), mask))
+				log.Info().Str("path", v.Selector.Jsonpath).Str("mask", mask.Name()).Msg("Add mask")
 				nbArg++
 			}
 		}
@@ -107,6 +110,7 @@ func BuildPipeline(pipeline Pipeline, conf Definition, caches map[string]Cache)
 			}
 			if present {
 				pipeline = pipeline.Process(NewMaskContextEngineProcess(NewPathSelector(v.Selector.Jsonpath), mask))
+				log.Info().Str("path", v.Selector.Jsonpath).Str("mask", mask.Name()).Msg("Add mask")
 				nbArg++
 			}
 		}

pkg/pipe/pipe.go

@@ -55,7 +55,7 @@ func NewMask(seed int64, injectParent string, injectRoot string, caches map[stri
 }
 
 func (me MaskEngine) MaskContext(e model.Dictionary, key string, context ...model.Dictionary) (model.Dictionary, error) {
-	log.Info().Msg("Mask pipe")
+	log.Debug().Msg("Mask pipe")
 	var result []model.Dictionary
 	input := []model.Dictionary{}
 
@@ -112,6 +112,10 @@ func (me MaskEngine) MaskContext(e model.Dictionary, key string, context ...mode
 	return copy, nil
 }
 
+func (me MaskEngine) Name() string {
+	return fmt.Sprintf("pipe injectParent=%s injectRoot=%s source=%s", me.injectParent, me.injectRoot, me.source)
+}
+
 // Factory create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskContextEngine, bool, error) {
 	if len(conf.Mask.Pipe.Masking) > 0 || len(conf.Mask.Pipe.DefinitionFile) > 0 {

pkg/randdate/randdate.go

@@ -18,6 +18,7 @@
 package randdate
 
 import (
+	"fmt"
 	"hash/fnv"
 	"math/rand"
 	"time"
@@ -41,12 +42,16 @@ func NewMask(min, max time.Time, seed int64) MaskEngine {
 
 // Mask choose a mask date randomly
 func (dateRange MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask randDate")
+	log.Debug().Msg("Mask randDate")
 	delta := dateRange.DateMax.Unix() - dateRange.DateMin.Unix()
 	sec := time.Unix(dateRange.rand.Int63n(delta)+dateRange.DateMin.Unix(), 0)
 	return sec, nil
 }
 
+func (dateRange MaskEngine) Name() string {
+	return fmt.Sprintf("randDate dateMin=%s dateMax=%s", dateRange.DateMin, dateRange.DateMax)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if conf.Mask.RandDate.DateMin != conf.Mask.RandDate.DateMax {

pkg/randdura/randdura.go

@@ -59,7 +59,7 @@ func NewMask(minString, maxString string, seed int64) (MaskEngine, error) {
 
 // Mask masks a time value with a duration
 func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
-	log.Info().Msg("Mask randomDuration")
+	log.Debug().Msg("Mask randomDuration")
 	var t time.Time
 	var err error
 	switch v := e.(type) {
@@ -88,6 +88,10 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 	return t.Add(time.Duration(dura)), nil
 }
 
+func (me MaskEngine) Name() string {
+	return fmt.Sprintf("randomDuration min=%s max=%s", me.Min, me.Max)
+}
+
 // Create a mask from a configuration
 func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
 	if len(conf.Mask.RandomDuration.Min) != 0 || len(conf.Mask.RandomDuration.Max) != 0 { // set differents seeds for differents jsonpath