Refactoring: change cache library (#309)

cmd/api/cache/ttl.go

@@ -4,106 +4,42 @@
 package cache
 
 import (
-	"context"
-	"sync"
 	"time"
 
-	"github.com/dgraph-io/badger/v4"
-	"github.com/rs/zerolog/log"
+	"github.com/cespare/xxhash"
+	"github.com/elastic/go-freelru"
 )
 
 type TTLCache struct {
-	db              *badger.DB
-	expiration      time.Duration
-	gcCollectPeriod time.Duration
-	wg              *sync.WaitGroup
+	db         *freelru.LRU[string, []byte]
+	expiration time.Duration
 }
 
-const dir = "/tmp/badger"
-
-func NewTTLCache(expiration time.Duration, inMemory bool) (*TTLCache, error) {
-	path := ""
-	if !inMemory {
-		path = dir
-	}
+func hashStringXXHASH(s string) uint32 {
+	return uint32(xxhash.Sum64String(s))
+}
 
-	db, err := badger.Open(badger.DefaultOptions(path).WithInMemory(inMemory))
+func NewTTLCache(expiration time.Duration) (*TTLCache, error) {
+	lru, err := freelru.New[string, []byte](8192*32, hashStringXXHASH)
 	if err != nil {
 		return nil, err
 	}
+	lru.SetLifetime(expiration)
 
 	return &TTLCache{
-		db:              db,
-		expiration:      expiration,
-		gcCollectPeriod: time.Minute * 5,
-		wg:              new(sync.WaitGroup),
+		db:         lru,
+		expiration: expiration,
 	}, nil
 }
 
-func (c *TTLCache) Start(ctx context.Context) {
-	c.wg.Add(1)
-	go c.gcCollect(ctx)
-}
-
-func (c *TTLCache) gcCollect(ctx context.Context) {
-	defer c.wg.Done()
-
-	ticker := time.NewTicker(c.gcCollectPeriod)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			if err := c.db.RunValueLogGC(0.5); err != nil {
-				log.Err(err).Msg("ttl cache garbage collection error")
-			}
-		}
-	}
-}
-
-func (c *TTLCache) Close() error {
-	c.wg.Wait()
-	return c.db.Close()
-}
-
 func (c *TTLCache) Get(key string) (data []byte, found bool) {
-	if err := c.db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(key))
-		if err != nil {
-			return err
-		}
-
-		if err := item.Value(func(val []byte) error {
-			data = make([]byte, len(val))
-			copy(data, val)
-			return nil
-		}); err != nil {
-			return err
-		}
-		return nil
-	}); err != nil {
-		return nil, false
-	}
-	return data, true
+	return c.db.Get(key)
 }
 
 func (c *TTLCache) Set(key string, data []byte) {
-	keyBytes := []byte(key)
-	err := c.db.Update(func(txn *badger.Txn) error {
-		e := badger.
-			NewEntry(keyBytes, data).
-			WithTTL(c.expiration)
-		return txn.SetEntry(e)
-	})
-	if err != nil {
-		log.Err(err).Msgf("set %s to TTL cache", key)
-	}
+	c.db.Add(key, data)
 }
 
 func (c *TTLCache) Clear() {
-	if err := c.db.DropAll(); err != nil {
-		log.Err(err).Msg("clear ttl cache")
-	}
+	c.db.Purge()
 }

cmd/api/cache/ttl_test.go

@@ -15,8 +15,7 @@ import (
 )
 
 func TestTTLCache_SetGet(t *testing.T) {
-	c, err := NewTTLCache(time.Second, true)
-	defer func() { _ = c.Close() }()
+	c, err := NewTTLCache(time.Second)
 	require.NoError(t, err)
 
 	t.Run("set and get key from cache", func(t *testing.T) {
@@ -103,8 +102,7 @@ func TestTTLCache_SetGet(t *testing.T) {
 }
 
 func TestTTLCache_Clear(t *testing.T) {
-	c, err := NewTTLCache(time.Second, true)
-	defer func() { _ = c.Close() }()
+	c, err := NewTTLCache(time.Second)
 	require.NoError(t, err)
 
 	t.Run("clear cache", func(t *testing.T) {

cmd/api/init.go

@@ -268,15 +268,11 @@ func initDatabase(cfg config.Database, viewsDir string) postgres.Storage {
 	return db
 }
 
-var ttlCache *cache.TTLCache
-
 func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Storage) {
-	var err error
-	ttlCache, err = cache.NewTTLCache(time.Minute*10, false)
+	ttlCache, err := cache.NewTTLCache(time.Minute * 30)
 	if err != nil {
 		panic(err)
 	}
-	ttlCache.Start(ctx)
 	ttlCacheMiddleware := cache.Middleware(ttlCache, nil)
 
 	if cfg.ApiConfig.Prometheus {

cmd/api/main.go

@@ -75,11 +75,6 @@ func main() {
 	if err := e.Shutdown(ctx); err != nil {
 		e.Logger.Fatal(err)
 	}
-	if ttlCache != nil {
-		if err := ttlCache.Close(); err != nil {
-			e.Logger.Fatal(err)
-		}
-	}
 	if gasTracker != nil {
 		if err := gasTracker.Close(); err != nil {
 			e.Logger.Fatal(err)

go.mod

@@ -16,12 +16,13 @@ require (
 	github.com/celestiaorg/go-square v1.1.1
 	github.com/celestiaorg/go-square/v2 v2.0.0
 	github.com/celestiaorg/rsmt2d v0.14.0
+	github.com/cespare/xxhash v1.1.0
 	github.com/cosmos/cosmos-sdk v0.46.16
 	github.com/cosmos/ibc-go/v6 v6.2.2
-	github.com/dgraph-io/badger/v4 v4.3.0
 	github.com/dipdup-io/workerpool v0.0.4
 	github.com/dipdup-net/go-lib v0.4.0
 	github.com/dipdup-net/indexer-sdk v0.0.5
+	github.com/elastic/go-freelru v0.15.0
 	github.com/fatih/structs v1.1.0
 	github.com/gabriel-vasile/mimetype v1.4.3
 	github.com/getsentry/sentry-go v0.27.0
@@ -93,7 +94,6 @@ require (
 	github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect
 	github.com/celestiaorg/nmt v0.22.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/cockroachdb/apd/v2 v2.0.2 // indirect
@@ -150,7 +150,6 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/btree v1.1.3 // indirect
-	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect

go.sum

@@ -451,6 +451,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY=
 github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
+github.com/elastic/go-freelru v0.15.0 h1:Jo1aY8JAvpyxbTDJEudrsBfjFDaALpfVv8mxuh9sfvI=
+github.com/elastic/go-freelru v0.15.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=