feat(chore): optimize encoded cache value

Signed-off-by: Mohammed Al Sahaf <[email protected]>

.golangci.yml

@@ -1,8 +1,6 @@
 run:
   go: '1.22.2'
 
-fix: true
-
 output:
   formats:
     - format: json
@@ -18,7 +16,6 @@ linters:
     - containedctx
     - cyclop
     - depguard
-    - err113
     - exhaustruct
     - forcetypeassert
     - funlen
@@ -29,7 +26,6 @@ linters:
     - interfacebloat
     - ireturn
     - lll
-    - mnd
     - nestif
     - nonamedreturns
     - paralleltest

Makefile

@@ -40,3 +40,6 @@ unit-tests:
 	for item in $(TESTS_LIST) ; do \
 		go test -v -race ./$$item ; \
 	done
+
+generate-protobuf:
+	buf generate

badger/badger.go

@@ -147,8 +147,8 @@ func (provider *Badger) ListKeys() []string {
 			_ = it.Item().Value(func(val []byte) error {
 				mapping, err := core.DecodeMapping(val)
 				if err == nil {
-					for _, v := range mapping.Mapping {
-						keys = append(keys, v.RealKey)
+					for _, v := range mapping.GetMapping() {
+						keys = append(keys, v.GetRealKey())
 					}
 				}
 

badger/caddy/go.mod

@@ -119,7 +119,7 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
 	google.golang.org/grpc v1.64.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v1.0.0 // indirect
 )

badger/caddy/go.sum

@@ -661,17 +661,15 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

badger/go.mod

@@ -28,7 +28,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )
 
 replace github.com/darkweak/storages/core => ../core

badger/go.sum

@@ -199,8 +199,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

buf.gen.yaml

@@ -0,0 +1,8 @@
+version: v2
+plugins:
+  - remote: buf.build/protocolbuffers/go:v1.31.0
+    out: core
+    opt:
+      - paths=source_relative
+inputs:
+  - directory: core

buf.yaml

@@ -0,0 +1,9 @@
+version: v2
+modules:
+  - path: core
+lint:
+  use:
+    - DEFAULT
+breaking:
+  use:
+    - FILE

core/core.go

@@ -3,28 +3,16 @@ package core
 import (
 	"bufio"
 	"bytes"
-	"encoding/gob"
 	"net/http"
 	"strings"
 	"time"
 
 	lz4 "github.com/pierrec/lz4/v4"
 	"go.uber.org/zap"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-type keyIndex struct {
-	StoredAt      time.Time   `json:"stored"`
-	FreshTime     time.Time   `json:"fresh"`
-	StaleTime     time.Time   `json:"stale"`
-	VariedHeaders http.Header `json:"varied"`
-	Etag          string      `json:"etag"`
-	RealKey       string      `json:"realKey"`
-}
-
-type StorageMapper struct {
-	Mapping map[string]keyIndex `json:"mapping"`
-}
-
 type Storer interface {
 	MapKeys(prefix string) map[string]string
 	ListKeys() []string
@@ -54,14 +42,14 @@ type CacheProvider struct {
 
 const MappingKeyPrefix = "IDX_"
 
-func DecodeMapping(item []byte) (mapping StorageMapper, e error) {
-	e = gob.NewDecoder(bytes.NewBuffer(item)).Decode(&mapping)
+func DecodeMapping(item []byte) (mapping *StorageMapper, e error) {
+	e = proto.Unmarshal(item, mapping)
 
 	return
 }
 
 func MappingElection(provider Storer, item []byte, req *http.Request, validator *Revalidator, logger *zap.Logger) (resultFresh *http.Response, resultStale *http.Response, e error) {
-	var mapping StorageMapper
+	var mapping *StorageMapper
 
 	if len(item) != 0 {
 		mapping, e = DecodeMapping(item)
@@ -70,11 +58,11 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
 		}
 	}
 
-	for keyName, keyItem := range mapping.Mapping {
+	for keyName, keyItem := range mapping.GetMapping() {
 		valid := true
 
-		for hname, hval := range keyItem.VariedHeaders {
-			if req.Header.Get(hname) != strings.Join(hval, ", ") {
+		for hname, hval := range keyItem.GetVariedHeaders() {
+			if req.Header.Get(hname) != strings.Join(hval.GetHeaderValue(), ", ") {
 				valid = false
 
 				break
@@ -85,11 +73,11 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
 			continue
 		}
 
-		ValidateETagFromHeader(keyItem.Etag, validator)
+		ValidateETagFromHeader(keyItem.GetEtag(), validator)
 
 		if validator.Matched {
 			// If the key is fresh enough.
-			if time.Since(keyItem.FreshTime) < 0 {
+			if time.Since(keyItem.GetFreshTime().AsTime()) < 0 {
 				response := provider.Get(keyName)
 				if response != nil {
 					bufW := new(bytes.Buffer)
@@ -109,7 +97,7 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
 			}
 
 			// If the key is still stale.
-			if time.Since(keyItem.StaleTime) < 0 {
+			if time.Since(keyItem.GetStaleTime().AsTime()) < 0 {
 				response := provider.Get(keyName)
 				if response != nil {
 					bufW := new(bytes.Buffer)
@@ -134,41 +122,46 @@ func MappingElection(provider Storer, item []byte, req *http.Request, validator
 }
 
 func MappingUpdater(key string, item []byte, logger *zap.Logger, now, freshTime, staleTime time.Time, variedHeaders http.Header, etag, realKey string) (val []byte, e error) {
-	var mapping StorageMapper
+	var mapping *StorageMapper
 	if len(item) == 0 {
-		mapping = StorageMapper{}
+		mapping = &StorageMapper{}
 	} else {
-		e = gob.NewDecoder(bytes.NewBuffer(item)).Decode(&mapping)
+		e = proto.Unmarshal(item, mapping)
 		if e != nil {
 			logger.Sugar().Errorf("Impossible to decode the key %s, %v", key, e)
 
 			return nil, e
 		}
 	}
 
-	if mapping.Mapping == nil {
-		mapping.Mapping = make(map[string]keyIndex)
+	if mapping.GetMapping() == nil {
+		mapping.Mapping = make(map[string]*KeyIndex)
+	}
+
+	var pbvariedeheader map[string]*KeyIndexStringList
+	if variedHeaders != nil {
+		pbvariedeheader = make(map[string]*KeyIndexStringList)
+	}
+
+	for k, v := range variedHeaders {
+		pbvariedeheader[k] = &KeyIndexStringList{HeaderValue: v}
 	}
 
-	mapping.Mapping[key] = keyIndex{
-		StoredAt:      now,
-		FreshTime:     freshTime,
-		StaleTime:     staleTime,
-		VariedHeaders: variedHeaders,
+	mapping.Mapping[key] = &KeyIndex{
+		StoredAt:      timestamppb.New(now),
+		FreshTime:     timestamppb.New(freshTime),
+		StaleTime:     timestamppb.New(staleTime),
+		VariedHeaders: pbvariedeheader,
 		Etag:          etag,
 		RealKey:       realKey,
 	}
 
-	buf := new(bytes.Buffer)
-
-	e = gob.NewEncoder(buf).Encode(mapping)
+	val, e = proto.Marshal(mapping)
 	if e != nil {
 		logger.Sugar().Errorf("Impossible to encode the mapping value for the key %s, %v", key, e)
 
 		return nil, e
 	}
 
-	val = buf.Bytes()
-
 	return val, e
 }

core/go.mod

@@ -5,9 +5,11 @@ go 1.22.1
 require (
 	github.com/pierrec/lz4/v4 v4.1.21
 	go.uber.org/zap v1.27.0
+	google.golang.org/protobuf v1.34.2
 )
 
 require (
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/stretchr/testify v1.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 )

core/go.sum

@@ -1,5 +1,6 @@
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -12,5 +13,7 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=