Migration: token metadata

internal/elastic/tokenmetadata/context.go

@@ -16,6 +16,9 @@ func buildGetTokenMetadataContext(ctx []tokenmetadata.GetContext, withSize bool)
 		if c.Network != "" {
 			filter = append(filter, core.Match("network", c.Network))
 		}
+		if c.Name != "" {
+			filter = append(filter, core.MatchPhrase("name", c.Name))
+		}
 		if c.MaxLevel > 0 {
 			filter = append(filter, core.Range("level", core.Item{"lte": c.MaxLevel}))
 		}

internal/helpers/string.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/btcsuite/btcutil/base58"
 	"github.com/google/uuid"
 )
 
@@ -51,3 +52,17 @@ func SpaceStringsBuilder(str string) string {
 func Slug(alias string) string {
 	return strings.ReplaceAll(strings.ToLower(alias), " ", "-")
 }
+
+// IsIPFS -
+func IsIPFS(hash string) bool {
+	if len(hash) != 46 && strings.HasPrefix(hash, "Qm") {
+		return false
+	}
+
+	data := base58.Decode(hash)
+	if len(data) == 34 && data[0] == 0x12 && data[1] == 0x20 {
+		return true
+	}
+
+	return false
+}

internal/helpers/string_test.go

@@ -28,3 +28,44 @@ func TestSlug(t *testing.T) {
 		})
 	}
 }
+
+func TestIsIPFS(t *testing.T) {
+	tests := []struct {
+		name string
+		hash string
+		want bool
+	}{
+		{
+			name: "QmSepFyoj44Xeok63ZVtSPehWg73dkGaUHAXyAwoeNyHj2",
+			hash: "QmSepFyoj44Xeok63ZVtSPehWg73dkGaUHAXyAwoeNyHj2",
+			want: true,
+		}, {
+			name: "QmX7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZy",
+			hash: "QmX7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZy",
+			want: true,
+		}, {
+			name: "QmaUx4RSLEiDrvKgb5NPtk1EHkcJHBQCWFcHQxTSep4VUw",
+			hash: "QmaUx4RSLEiDrvKgb5NPtk1EHkcJHBQCWFcHQxTSep4VUw",
+			want: true,
+		}, {
+			name: "QmX7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZ",
+			hash: "QmX7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZ",
+			want: false,
+		}, {
+			name: "Q1X7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZ",
+			hash: "Q1X7Bhf8Gw7BDdyyGLPrsdfyf9pyqgAY4M3BzzSVwULxZ",
+			want: false,
+		}, {
+			name: "undefined",
+			hash: "undefined",
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsIPFS(tt.hash); got != tt.want {
+				t.Errorf("IsIPFS() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/logger/logger.go

@@ -189,3 +189,8 @@ func With(entry Loggable) *logrus.Entry {
 func WithNetwork(network string) *logrus.Entry {
 	return logger.WithField("network", network)
 }
+
+// WithField -
+func WithField(field string, value interface{}) *logrus.Entry {
+	return logger.WithField(field, value)
+}

internal/models/tokenmetadata/context.go

@@ -8,4 +8,5 @@ type GetContext struct {
 	MaxLevel int64
 	MinLevel int64
 	Creator  string
+	Name     string
 }

scripts/migration/main.go

@@ -29,6 +29,7 @@ var migrationsList = []migrations.Migration{
 	&migrations.TokenBalanceRecalc{},
 	&migrations.TokenMetadataSetDecimals{},
 	&migrations.NFTMetadata{},
+	&migrations.TokenMetadataUnknown{},
 }
 
 func main() {

scripts/migration/migrations/token_metadata_unknown.go

@@ -0,0 +1,93 @@
+package migrations
+
+import (
+	"errors"
+	"strings"
+	"time"
+
+	"github.com/baking-bad/bcdhub/internal/config"
+	"github.com/baking-bad/bcdhub/internal/helpers"
+	"github.com/baking-bad/bcdhub/internal/logger"
+	"github.com/baking-bad/bcdhub/internal/models"
+	"github.com/baking-bad/bcdhub/internal/models/tokenmetadata"
+	tzipStorage "github.com/baking-bad/bcdhub/internal/parsers/tzip/storage"
+	"github.com/baking-bad/bcdhub/internal/parsers/tzip/tokens"
+	"github.com/schollz/progressbar/v3"
+)
+
+// TokenMetadataUnknown - migration that requests again token metadata
+type TokenMetadataUnknown struct{}
+
+// Key -
+func (m *TokenMetadataUnknown) Key() string {
+	return "token_metadata_unknown"
+}
+
+// Description -
+func (m *TokenMetadataUnknown) Description() string {
+	return "migration that requests again token metadata"
+}
+
+// Do - migrate function
+func (m *TokenMetadataUnknown) Do(ctx *config.Context) error {
+	metadata, err := ctx.TokenMetadata.GetAll(tokenmetadata.GetContext{
+		Name: "Unknown",
+	})
+	if err != nil {
+		return err
+	}
+	logger.Info("Found %d unknown metadata", len(metadata))
+
+	bar := progressbar.NewOptions(len(metadata), progressbar.OptionSetPredictTime(false), progressbar.OptionClearOnFinish(), progressbar.OptionShowCount())
+
+	updates := make([]models.Model, 0)
+	for i := range metadata {
+		if err := bar.Add(1); err != nil {
+			return err
+		}
+
+		emptyValue, ok := metadata[i].Extras["@@empty"]
+		if !ok {
+			continue
+		}
+		link, ok := emptyValue.(string)
+		if !ok {
+			continue
+		}
+
+		if !helpers.IsIPFS(strings.TrimPrefix(link, "ipfs://")) {
+			continue
+		}
+
+		s := tzipStorage.NewIPFSStorage(ctx.Config.IPFSGateways, tzipStorage.WithTimeoutIPFS(time.Second*10))
+
+		remoteMetadata := new(tokens.TokenMetadata)
+		if err := s.Get(link, remoteMetadata); err != nil {
+			if errors.Is(err, tzipStorage.ErrNoIPFSResponse) {
+				logger.WithField("url", link).WithField("kind", "token_metadata").Warning(err)
+				continue
+			}
+			return err
+		}
+		metadata[i].Symbol = remoteMetadata.Symbol
+		metadata[i].Decimals = remoteMetadata.Decimals
+		metadata[i].Name = remoteMetadata.Name
+		metadata[i].Description = remoteMetadata.Description
+		metadata[i].ArtifactURI = remoteMetadata.ArtifactURI
+		metadata[i].DisplayURI = remoteMetadata.DisplayURI
+		metadata[i].ThumbnailURI = remoteMetadata.ThumbnailURI
+		metadata[i].ExternalURI = remoteMetadata.ExternalURI
+		metadata[i].IsTransferable = remoteMetadata.IsTransferable
+		metadata[i].IsBooleanAmount = remoteMetadata.IsBooleanAmount
+		metadata[i].ShouldPreferSymbol = remoteMetadata.ShouldPreferSymbol
+		metadata[i].Creators = remoteMetadata.Creators
+		metadata[i].Tags = remoteMetadata.Tags
+		metadata[i].Formats = remoteMetadata.Formats
+		metadata[i].Extras = remoteMetadata.Extras
+
+		updates = append(updates, &metadata[i])
+	}
+
+	return ctx.Storage.BulkInsert(updates)
+
+}