perf: pgo and json serialisation optimizations (#325)

* Add files via upload

* perf: fast json serialization

* style: lint

* docs: CHANGELOG.md

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM adrienaury/go-devcontainer:v2.0
+FROM adrienaury/go-devcontainer:v4.1
 
 USER root
 

CHANGELOG.md

@@ -14,6 +14,10 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [1.26.1]
+
+- `Fixed` performance issues on JSON serialization
+
 ## [1.26.0]
 
 - `Added` property `preserve-list` to masking definition to be able to ignore masking specific values

go.mod

@@ -1,13 +1,17 @@
 module github.com/cgi-fr/pimo
 
-go 1.20
+go 1.21
+
+toolchain go1.22.2
 
 require (
 	github.com/CGI-FR/xixo v0.1.8
 	github.com/Masterminds/sprig/v3 v3.3.0
 	github.com/adrienaury/zeromdc v0.1.1
 	github.com/capitalone/fpe v1.2.1
+	github.com/goccy/go-json v0.10.3
 	github.com/goccy/go-yaml v1.12.0
+	github.com/iancoleman/orderedmap v0.3.0
 	github.com/invopop/jsonschema v0.12.0
 	github.com/kenshaw/baseconv v0.1.1
 	github.com/labstack/echo/v4 v4.12.0
@@ -19,7 +23,6 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.9.0
 	github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea
-	gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a
 	golang.org/x/crypto v0.27.0
 	golang.org/x/exp v0.0.0-20240707233637-46b078467d37
 	golang.org/x/text v0.18.0

go.sum

@@ -29,15 +29,22 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM=
 github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo=
 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
@@ -48,6 +55,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
+github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
@@ -59,12 +68,15 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/kenshaw/baseconv v0.1.1 h1:oAu/C7ipUT2PqT9DT0mZDGDg4URIglizZMjPv9oCu0E=
 github.com/kenshaw/baseconv v0.1.1/go.mod h1:yy9zGmnnR6vgOxOQb702nVdAG30JhyYZpj/5/m0siRI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
 github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
 github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
 github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/anko v0.1.9 h1:OpdRDvjOY/FQXoGXkN0n7b0br4a4fHB5hhsaq0Cd3ds=
@@ -88,6 +100,7 @@ github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDj
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
@@ -118,8 +131,6 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI=
 github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4=
-gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a h1:DxppxFKRqJ8WD6oJ3+ZXKDY0iMONQDl5UTg2aTyHh8k=
-gitlab.com/c0b/go-ordered-json v0.0.0-20201030195603-febf46534d5a/go.mod h1:NREvu3a57BaK0R1+ztrEzHWiZAihohNLQ6trPxlIqZI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
 golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=

pkg/model/ordered_dict.go

@@ -1,11 +1,14 @@
 package model
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 
 	"github.com/rs/zerolog/log"
-	"gitlab.com/c0b/go-ordered-json"
+
+	goccy "github.com/goccy/go-json"
+	ordered "github.com/iancoleman/orderedmap"
 )
 
 type Dictionary struct {
@@ -17,7 +20,7 @@ func NewPackedDictionary() Dictionary {
 }
 
 func NewDictionary() Dictionary {
-	return Dictionary{ordered.NewOrderedMap()}
+	return Dictionary{ordered.New()}
 }
 
 func Copy(other Entry) Entry {
@@ -37,19 +40,15 @@ func CopyDictionary(other Dictionary) Dictionary {
 		return NewDictionary()
 	}
 
-	om := ordered.NewOrderedMap()
-	iter := other.EntriesIter()
+	om := ordered.New()
 
-	for {
-		pair, ok := iter()
-		if !ok {
-			break
-		}
-		switch typedVal := pair.Value.(type) {
+	for _, key := range other.Keys() {
+		value := other.Get(key)
+		switch typedVal := value.(type) {
 		case Dictionary:
-			om.Set(pair.Key, CopyDictionary(typedVal))
+			om.Set(key, CopyDictionary(typedVal))
 		default:
-			om.Set(pair.Key, pair.Value)
+			om.Set(key, value)
 		}
 	}
 	return Dictionary{om}
@@ -72,31 +71,29 @@ func CleanTypes(inter interface{}) interface{} {
 		}
 		return dict
 	case *Dictionary:
-		iter := typedInter.EntriesIter()
 		dict := NewDictionary()
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			dict.Set(pair.Key, CleanTypes(pair.Value))
+		for _, key := range typedInter.Keys() {
+			dict.Set(key, CleanTypes(typedInter.Get(key)))
 		}
 		return dict
 	case Dictionary:
-		iter := typedInter.EntriesIter()
 		dict := NewDictionary()
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			dict.Set(pair.Key, CleanTypes(pair.Value))
+		for _, key := range typedInter.Keys() {
+			dict.Set(key, CleanTypes(typedInter.Get(key)))
 		}
 		return dict
 	case *ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		dict := NewDictionary()
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			dict.Set(pair.Key, CleanTypes(pair.Value))
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			dict.Set(key, CleanTypes(value))
 		}
 		return dict
 	case ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		dict := NewDictionary()
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			dict.Set(pair.Key, CleanTypes(pair.Value))
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			dict.Set(key, CleanTypes(value))
 		}
 		return dict
 	case []interface{}:
@@ -165,31 +162,29 @@ func Untyped(inter interface{}) interface{} {
 		}
 		return cleanmap
 	case *Dictionary:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]interface{}{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = Untyped(pair.Value)
+		for _, key := range typedInter.Keys() {
+			cleanmap[key] = Untyped(typedInter.Get(key))
 		}
 		return cleanmap
 	case Dictionary:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]interface{}{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = Untyped(pair.Value)
+		for _, key := range typedInter.Keys() {
+			cleanmap[key] = Untyped(typedInter.Get(key))
 		}
 		return cleanmap
 	case *ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]interface{}{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = Untyped(pair.Value)
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			cleanmap[key] = Untyped(value)
 		}
 		return cleanmap
 	case ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]interface{}{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = Untyped(pair.Value)
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			cleanmap[key] = Untyped(value)
 		}
 		return cleanmap
 	case []interface{}:
@@ -234,31 +229,29 @@ func UnorderedTypes(inter interface{}) interface{} {
 		}
 		return cleanmap
 	case *Dictionary:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]Entry{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = UnorderedTypes(pair.Value)
+		for _, key := range typedInter.Keys() {
+			cleanmap[key] = UnorderedTypes(typedInter.Get(key))
 		}
 		return cleanmap
 	case Dictionary:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]Entry{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = UnorderedTypes(pair.Value)
+		for _, key := range typedInter.Keys() {
+			cleanmap[key] = UnorderedTypes(typedInter.Get(key))
 		}
 		return cleanmap
 	case *ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]Entry{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = UnorderedTypes(pair.Value)
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			cleanmap[key] = UnorderedTypes(value)
 		}
 		return cleanmap
 	case ordered.OrderedMap:
-		iter := typedInter.EntriesIter()
 		cleanmap := map[string]Entry{}
-		for pair, ok := iter(); ok; pair, ok = iter() {
-			cleanmap[pair.Key] = UnorderedTypes(pair.Value)
+		for _, key := range typedInter.Keys() {
+			value, _ := typedInter.Get(key)
+			cleanmap[key] = UnorderedTypes(value)
 		}
 		return cleanmap
 	case []interface{}:
@@ -294,7 +287,7 @@ func UnorderedTypes(inter interface{}) interface{} {
 }
 
 func (d Dictionary) IsPacked() bool {
-	_, packed := d.GetValue(".")
+	_, packed := d.OrderedMap.Get(".")
 	return packed
 }
 
@@ -353,3 +346,36 @@ func (d Dictionary) With(key string, value interface{}) Dictionary {
 	result.Set(key, CleanTypes(value))
 	return result
 }
+
+func (d Dictionary) GetValue(key string) (Entry, bool) {
+	return d.OrderedMap.Get(key)
+}
+
+func (d Dictionary) Get(key string) Entry {
+	entry, _ := d.OrderedMap.Get(key)
+	return entry
+}
+
+func (d Dictionary) MarshalJSON() ([]byte, error) {
+	var buf bytes.Buffer
+	buf.WriteByte('{')
+	encoder := goccy.NewEncoder(&buf)
+	for i, k := range d.Keys() {
+		if i > 0 {
+			buf.WriteByte(',')
+		}
+		// add key
+		if err := encoder.Encode(k); err != nil {
+			return nil, err
+		}
+		buf.Truncate(buf.Len() - 1) // remove last new line
+		buf.WriteByte(':')
+		// add value
+		if err := encoder.Encode(d.Get(k)); err != nil {
+			return nil, err
+		}
+		buf.Truncate(buf.Len() - 1) // remove last new line
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}

pkg/model/process_mask.go

@@ -77,7 +77,7 @@ func (mep *MaskEngineProcess) ProcessDictionary(dictionary Dictionary, out Colle
 
 	if ret == nil {
 		out.Collect(result)
-		return
+		return nil
 	}
 
 	if ret != nil && skipLineOnError {

pkg/timeline/timeline.go

@@ -131,14 +131,8 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 
 	initialState := map[string]*int64{}
 	if dict, ok := e.(model.Dictionary); ok {
-		iter := dict.EntriesIter()
-		for {
-			pair, ok := iter()
-			if !ok {
-				break
-			}
-
-			initialState[pair.Key] = me.formatTimestamp(pair.Value)
+		for _, key := range dict.Keys() {
+			initialState[key] = me.formatTimestamp(dict.Get(key))
 		}
 	}