Validate IPFS multihash (#763)

cmd/metrics/main.go

@@ -48,7 +48,7 @@ func main() {
 
 	workers := []services.Service{
 		services.NewView(ctx.StorageDB.DB, "head_stats", time.Minute),
-		services.NewUnknown(ctx, time.Hour, time.Second*2),
+		services.NewUnknown(ctx, time.Minute*30, time.Second*2, -time.Hour*24),
 		services.NewStorageBased(
 			"projects",
 			ctx.Services,

cmd/metrics/services/unknown.go

@@ -1,12 +1,10 @@
 package services
 
 import (
-	"strings"
 	"time"
 
 	"github.com/baking-bad/bcdhub/internal/bcd/consts"
 	"github.com/baking-bad/bcdhub/internal/config"
-	"github.com/baking-bad/bcdhub/internal/helpers"
 	"github.com/baking-bad/bcdhub/internal/logger"
 	"github.com/baking-bad/bcdhub/internal/models/tokenmetadata"
 	"github.com/baking-bad/bcdhub/internal/models/types"
@@ -21,26 +19,31 @@ type Unknown struct {
 	*TimeBased
 	ctx     *config.Context
 	timeout time.Duration
+	since   time.Duration
 }
 
 // NewUnknown -
-func NewUnknown(ctx *config.Context, period time.Duration, timeout time.Duration) *Unknown {
+func NewUnknown(ctx *config.Context, period, timeout, since time.Duration) *Unknown {
 	u := &Unknown{
 		ctx:     ctx,
 		timeout: timeout,
+		since:   since,
 	}
 	u.TimeBased = NewTimeBased(u.refresh, period)
 	return u
 }
 
 func (u *Unknown) refresh() error {
-	metadata, err := u.ctx.TokenMetadata.GetAll(tokenmetadata.GetContext{
+	since := time.Now().Add(u.since)
+	metadata, err := u.ctx.TokenMetadata.GetRecent(since, tokenmetadata.GetContext{
 		Name: consts.Unknown,
 	})
 	if err != nil {
 		return err
 	}
-	logger.Info().Msgf("Found %d unknown metadata", len(metadata))
+	logger.Info().Msgf("Found %d unknown token metadata", len(metadata))
+
+	ipfs := tzipStorage.NewIPFSStorage(u.ctx.Config.IPFSGateways, tzipStorage.WithTimeoutIPFS(u.timeout))
 
 	return u.ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
 		for i := range metadata {
@@ -53,16 +56,9 @@ func (u *Unknown) refresh() error {
 				continue
 			}
 
-			if !helpers.IsIPFS(strings.TrimPrefix(link, "ipfs://")) {
-				continue
-			}
-
-			s := tzipStorage.NewIPFSStorage(u.ctx.Config.IPFSGateways, tzipStorage.WithTimeoutIPFS(u.timeout))
-
 			remoteMetadata := new(tokens.TokenMetadata)
-			if err := s.Get(link, remoteMetadata); err != nil {
-				if errors.Is(err, tzipStorage.ErrNoIPFSResponse) {
-					logger.Warning().Err(err).Str("url", link).Str("kind", "token_metadata").Msg("")
+			if err := ipfs.Get(link, remoteMetadata); err != nil {
+				if errors.Is(err, tzipStorage.ErrNoIPFSResponse) || errors.Is(err, tzipStorage.ErrInvalidIPFSHash) {
 					continue
 				}
 				return err
@@ -86,6 +82,7 @@ func (u *Unknown) refresh() error {
 			if err := metadata[i].Save(tx); err != nil {
 				return err
 			}
+			logger.Info().Str("url", link).Msg("token metadata fetched")
 		}
 
 		return nil

internal/config/config.go

@@ -216,7 +216,7 @@ func LoadConfig(filename string) (Config, error) {
 	return config, nil
 }
 
-var defaultEnv = regexp.MustCompile(`\${(?P<name>[\w\.]{1,}):-(?P<value>[\w\.:/]*)}`)
+var defaultEnv = regexp.MustCompile(`\${(?P<name>[\w\.]{1,}):-(?P<value>[\w\.:/-]*)}`)
 
 func expandEnv(data string) string {
 	vars := defaultEnv.FindAllStringSubmatch(data, -1)

internal/config/config_test.go

@@ -22,12 +22,12 @@ func Test_expandEnv(t *testing.T) {
 			want: "val ",
 		}, {
 			name: "test 4",
-			data: "${TEST3:-127.0.0.1}",
+			data: "${TEST4:-127.0.0.1}",
 			want: "127.0.0.1",
 		}, {
 			name: "test 5",
-			data: "${TEST3:-https://example.com:443/path/}",
-			want: "127.0.0.1",
+			data: "${TEST5:-https://example-site.com:443/path/}",
+			want: "https://example-site.com:443/path/",
 		},
 	}
 	for _, tt := range tests {

internal/models/tokenmetadata/repository.go

@@ -1,12 +1,17 @@
 package tokenmetadata
 
-import "github.com/baking-bad/bcdhub/internal/models/types"
+import (
+	"time"
+
+	"github.com/baking-bad/bcdhub/internal/models/types"
+)
 
 // Repository -
 type Repository interface {
 	Get(ctx []GetContext, size, offset int64) ([]TokenMetadata, error)
 	GetAll(ctx ...GetContext) ([]TokenMetadata, error)
 	GetOne(network types.Network, contract string, tokenID uint64) (*TokenMetadata, error)
+	GetRecent(since time.Time, ctx ...GetContext) ([]TokenMetadata, error)
 	GetWithExtras() ([]TokenMetadata, error)
 	Count(ctx []GetContext) (int64, error)
 }

internal/parsers/tzip/storage/errors.go

@@ -13,4 +13,5 @@ var (
 	ErrHTTPRequest               = errors.New("HTTP request error")
 	ErrJSONDecoding              = errors.New("JSON decoding error")
 	ErrNoIPFSResponse            = errors.New("can't load document from IPFS")
+	ErrInvalidIPFSHash           = errors.New("Invalid IPFS multihash")
 )

internal/parsers/tzip/storage/ipfs.go

@@ -3,20 +3,32 @@ package storage
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/baking-bad/bcdhub/internal/logger"
+	"github.com/karlseguin/ccache"
 )
 
 // IPFS storage prefix
 const (
 	PrefixIPFS = "ipfs"
+	MaxTTL     = time.Duration(1<<63 - 1)
+	BounceTime = time.Minute * 5
+	BounceFlag = "no_response"
+)
+
+var (
+	regMultihash = regexp.MustCompile("Qm[1-9A-HJ-NP-Za-km-z]{44}")
 )
 
 // IPFSStorage -
 type IPFSStorage struct {
 	HTTPStorage
 	gateways []string
+	cache    *ccache.Cache
 }
 
 // IPFSStorageOption -
@@ -42,6 +54,7 @@ func NewIPFSStorage(gateways []string, opts ...IPFSStorageOption) IPFSStorage {
 	s := IPFSStorage{
 		HTTPStorage: NewHTTPStorage(),
 		gateways:    gateways,
+		cache:       ccache.New(ccache.Configure()),
 	}
 
 	for i := range opts {
@@ -57,12 +70,30 @@ func (s IPFSStorage) Get(value string, output interface{}) error {
 		return ErrEmptyIPFSGatewayList
 	}
 
+	multihash := strings.TrimPrefix(value, "ipfs://")
+	if len(multihash) != 46 || !regMultihash.MatchString(multihash) {
+		return ErrInvalidIPFSHash
+	}
+
+	if item := s.cache.Get(multihash); item != nil && !item.Expired() {
+		output = item.Value()
+		logger.Info().Str("url", value).Msg("using cached response")
+		if output == BounceFlag {
+			return ErrNoIPFSResponse
+		}
+		return nil
+	}
+
 	for i := range s.gateways {
-		url := fmt.Sprintf("%s/ipfs/%s", s.gateways[i], strings.TrimPrefix(value, "ipfs://"))
-		if err := s.HTTPStorage.Get(url, output); err == nil {
+		url := fmt.Sprintf("%s/ipfs/%s", s.gateways[i], multihash)
+		err := s.HTTPStorage.Get(url, output)
+		if err == nil {
+			s.cache.Set(multihash, output, MaxTTL)
 			return nil
 		}
+		logger.Warning().Err(err).Str("url", url).Msg("")
 	}
 
+	s.cache.Set(multihash, BounceFlag, BounceTime)
 	return ErrNoIPFSResponse
 }

internal/postgres/tokenmetadata/storage.go

@@ -2,6 +2,7 @@ package tokenmetadata
 
 import (
 	"errors"
+	"time"
 
 	"github.com/baking-bad/bcdhub/internal/models"
 	"github.com/baking-bad/bcdhub/internal/models/tokenmetadata"
@@ -54,6 +55,17 @@ func (storage *Storage) GetAll(ctx ...tokenmetadata.GetContext) (tokens []tokenm
 	return
 }
 
+// GetRecent -
+func (storage *Storage) GetRecent(since time.Time, ctx ...tokenmetadata.GetContext) (tokens []tokenmetadata.TokenMetadata, err error) {
+	query := storage.DB.Table(models.DocTokenMetadata)
+	storage.buildGetTokenMetadataContext(query, ctx...)
+	err = query.
+		Where("timestamp > ?", since).
+		Order("id desc").
+		Find(&tokens).Error
+	return
+}
+
 // GetWithExtras -
 func (storage *Storage) GetWithExtras() (tokens []tokenmetadata.TokenMetadata, err error) {
 	err = storage.DB.Table(models.DocTokenMetadata).

scripts/migration/migrations/token_metadata_unknown.go

@@ -2,12 +2,10 @@ package migrations
 
 import (
 	"errors"
-	"strings"
 	"time"
 
 	"github.com/baking-bad/bcdhub/internal/bcd/consts"
 	"github.com/baking-bad/bcdhub/internal/config"
-	"github.com/baking-bad/bcdhub/internal/helpers"
 	"github.com/baking-bad/bcdhub/internal/logger"
 	"github.com/baking-bad/bcdhub/internal/models/tokenmetadata"
 	"github.com/baking-bad/bcdhub/internal/models/types"
@@ -32,14 +30,15 @@ func (m *TokenMetadataUnknown) Description() string {
 
 // Do - migrate function
 func (m *TokenMetadataUnknown) Do(ctx *config.Context) error {
-	metadata, err := ctx.TokenMetadata.GetAll(tokenmetadata.GetContext{
+	metadata, err := ctx.TokenMetadata.GetRecent(time.Time{}, tokenmetadata.GetContext{
 		Name: consts.Unknown,
 	})
 	if err != nil {
 		return err
 	}
 	logger.Info().Msgf("Found %d unknown metadata", len(metadata))
 
+	ipfs := tzipStorage.NewIPFSStorage(ctx.Config.IPFSGateways, tzipStorage.WithTimeoutIPFS(time.Second*5))
 	bar := progressbar.NewOptions(len(metadata), progressbar.OptionSetPredictTime(false), progressbar.OptionClearOnFinish(), progressbar.OptionShowCount())
 
 	return ctx.StorageDB.DB.Transaction(func(tx *gorm.DB) error {
@@ -57,15 +56,9 @@ func (m *TokenMetadataUnknown) Do(ctx *config.Context) error {
 				continue
 			}
 
-			if !helpers.IsIPFS(strings.TrimPrefix(link, "ipfs://")) {
-				continue
-			}
-
-			s := tzipStorage.NewIPFSStorage(ctx.Config.IPFSGateways, tzipStorage.WithTimeoutIPFS(time.Second*10))
-
 			remoteMetadata := new(tokens.TokenMetadata)
-			if err := s.Get(link, remoteMetadata); err != nil {
-				if errors.Is(err, tzipStorage.ErrNoIPFSResponse) {
+			if err := ipfs.Get(link, remoteMetadata); err != nil {
+				if errors.Is(err, tzipStorage.ErrNoIPFSResponse) || errors.Is(err, tzipStorage.ErrInvalidIPFSHash) {
 					logger.Warning().Err(err).Str("url", link).Str("kind", "token_metadata").Msg("")
 					continue
 				}