From 1e5d6a1ad077e5788f04fd7058a5f5f41c7c234c Mon Sep 17 00:00:00 2001 From: Roman Serikov Date: Tue, 10 Nov 2020 19:29:06 +0300 Subject: [PATCH] move deployment and verification metrics to compiler --- cmd/api/handlers/deployment.go | 23 -------- cmd/compiler/main.go | 98 +++++++++++++++++++++++++++++++-- cmd/compiler/verification.go | 8 +-- cmd/metrics/contract.go | 6 -- cmd/metrics/operations.go | 4 -- internal/database/database.go | 13 +++-- internal/database/deployment.go | 21 +++++-- internal/metrics/contract.go | 21 ------- internal/metrics/operation.go | 63 --------------------- 9 files changed, 119 insertions(+), 138 deletions(-) diff --git a/cmd/api/handlers/deployment.go b/cmd/api/handlers/deployment.go index 5b21dbe32..b60250269 100644 --- a/cmd/api/handlers/deployment.go +++ b/cmd/api/handlers/deployment.go @@ -10,9 +10,7 @@ import ( "github.com/baking-bad/bcdhub/internal/compiler/compilation" "github.com/baking-bad/bcdhub/internal/compiler/filesgenerator" "github.com/baking-bad/bcdhub/internal/database" - "github.com/baking-bad/bcdhub/internal/elastic" "github.com/baking-bad/bcdhub/internal/logger" - "github.com/baking-bad/bcdhub/internal/metrics" "github.com/gin-gonic/gin" ) @@ -151,26 +149,5 @@ func (ctx *Context) FinalizeDeployment(c *gin.Context) { return } - op, err := ctx.ES.GetOperations( - map[string]interface{}{ - "hash": req.OperationHash, - }, - 0, - true, - ) - - if !elastic.IsRecordNotFound(err) && handleError(c, err, 0) { - return - } - - if len(op) != 0 { - h := metrics.New(ctx.ES, ctx.DB) - - err := h.SetOperationDeployment(&op[0]) - if handleError(c, err, 0) { - return - } - } - c.JSON(http.StatusOK, gin.H{"status": compilation.StatusSuccess}) } diff --git a/cmd/compiler/main.go b/cmd/compiler/main.go index 3af10529f..06a0fde31 100644 --- a/cmd/compiler/main.go +++ b/cmd/compiler/main.go @@ -7,11 +7,16 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/baking-bad/bcdhub/internal/compiler/compilation" "github.com/baking-bad/bcdhub/internal/config" + "github.com/baking-bad/bcdhub/internal/contractparser/consts" + "github.com/baking-bad/bcdhub/internal/database" + "github.com/baking-bad/bcdhub/internal/elastic" "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/logger" + "github.com/baking-bad/bcdhub/internal/models" "github.com/baking-bad/bcdhub/internal/mq" "github.com/streadway/amqp" ) @@ -43,16 +48,21 @@ func main() { ), } - msgs, err := context.MQ.Consume(mq.QueueCompilations) - if err != nil { - logger.Fatal(err) - } - defer context.Close() signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + protocol, err := context.ES.GetProtocol(consts.Mainnet, "", -1) + if err != nil { + log.Fatal(err) + } + ticker := time.NewTicker(time.Second * time.Duration(protocol.Constants.TimeBetweenBlocks)) + + msgs, err := context.MQ.Consume(mq.QueueCompilations) + if err != nil { + logger.Fatal(err) + } logger.Info("Connected to %s queue", mq.QueueCompilations) for { @@ -60,6 +70,10 @@ func main() { case <-signals: logger.Info("Stopped compiler") return + case <-ticker.C: + if err := context.setDeployment(); err != nil { + logger.Error(err) + } case msg := <-msgs: if err := context.handleMessage(msg); err != nil { logger.Error(err) @@ -69,6 +83,80 @@ func main() { } +func (ctx *Context) setDeployment() error { + deployments, err := ctx.DB.GetDeploymentsByAddressNetwork("", "") + if err != nil { + return err + } + + for i, d := range deployments { + ops, err := ctx.ES.GetOperations( + map[string]interface{}{"hash": d.OperationHash}, + 0, + true, + ) + + if err != nil { + if elastic.IsRecordNotFound(err) { + continue + } + + return fmt.Errorf("GetOperations %s error %w", d.OperationHash, err) + } + + if len(ops) == 0 { + continue + } + + if err := ctx.processDeployment(&deployments[i], &ops[0]); err != nil { + return fmt.Errorf("deployment ID %d operationHash %s processDeployment error %w", d.ID, d.OperationHash, err) + } + } + + return nil +} + +func (ctx *Context) processDeployment(deployment *database.Deployment, operation *models.Operation) error { + deployment.Address = operation.Destination + deployment.Network = operation.Network + + if err := ctx.DB.UpdateDeployment(deployment); err != nil { + return fmt.Errorf("UpdateDeployment error %w", err) + } + + task, err := ctx.DB.GetCompilationTask(deployment.CompilationTaskID) + if err != nil { + return fmt.Errorf("task ID %d GetCompilationTask error %w", deployment.CompilationTaskID, err) + } + + var sourcePath string + + for _, r := range task.Results { + if r.Status == compilation.StatusSuccess { + sourcePath = r.AWSPath + break + } + } + + verification := database.Verification{ + UserID: task.UserID, + CompilationTaskID: deployment.CompilationTaskID, + Address: operation.Destination, + Network: operation.Network, + SourcePath: sourcePath, + } + + if err := ctx.DB.CreateVerification(&verification); err != nil { + return fmt.Errorf("CreateVerification error %w", err) + } + + contract := models.NewEmptyContract(task.Network, task.Address) + contract.Verified = true + contract.VerificationSource = sourcePath + + return ctx.ES.UpdateFields(elastic.DocContracts, contract.GetID(), contract, "Verified", "VerificationSource") +} + func (ctx *Context) handleMessage(data amqp.Delivery) error { if err := ctx.parseData(data); err != nil { return err diff --git a/cmd/compiler/verification.go b/cmd/compiler/verification.go index 6f0939a6d..52753c9f6 100644 --- a/cmd/compiler/verification.go +++ b/cmd/compiler/verification.go @@ -5,6 +5,7 @@ import ( "github.com/baking-bad/bcdhub/internal/compiler/compilation" "github.com/baking-bad/bcdhub/internal/database" + "github.com/baking-bad/bcdhub/internal/elastic" "github.com/baking-bad/bcdhub/internal/helpers" "github.com/baking-bad/bcdhub/internal/logger" "github.com/baking-bad/bcdhub/internal/models" @@ -59,11 +60,10 @@ func (ctx *Context) verification(ct compilation.Task) error { } contract := models.NewEmptyContract(task.Network, task.Address) - if err := ctx.ES.GetByID(&contract); err != nil { - return err - } + contract.Verified = true + contract.VerificationSource = sourcePath - return ctx.MQ.Send(&contract) + return ctx.ES.UpdateFields(elastic.DocContracts, contract.GetID(), contract, "Verified", "VerificationSource") } func (ctx *Context) verify(ct compilation.Task) (*database.CompilationTask, error) { diff --git a/cmd/metrics/contract.go b/cmd/metrics/contract.go index 2b0a015de..d0befad40 100644 --- a/cmd/metrics/contract.go +++ b/cmd/metrics/contract.go @@ -31,12 +31,6 @@ func parseContract(contract *models.Contract) error { h.SetContractAlias(ctx.Aliases, contract) } - if !contract.Verified { - if err := h.SetContractVerification(contract); err != nil { - return errors.Errorf("[parseContract] Error during set contract verification: %s", err) - } - } - rpc, err := ctx.GetRPC(contract.Network) if err != nil { return err diff --git a/cmd/metrics/operations.go b/cmd/metrics/operations.go index 9ba1956b4..1c6bd8975 100644 --- a/cmd/metrics/operations.go +++ b/cmd/metrics/operations.go @@ -41,10 +41,6 @@ func parseOperation(h *metrics.Handler, operation models.Operation) error { h.SetOperationAliases(ctx.Aliases, &operation) h.SetOperationStrings(&operation) - if err := h.SetOperationDeployment(&operation); err != nil { - return err - } - if helpers.IsContract(operation.Destination) || operation.IsOrigination() { if err := h.SendSentryNotifications(operation); err != nil { return err diff --git a/internal/database/database.go b/internal/database/database.go index 9c4d72fb1..2b974bfce 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -32,7 +32,7 @@ type IAccount interface { type IAssessment interface { CreateAssessment(a *Assessments) error CreateOrUpdateAssessment(a *Assessments) error - GetAssessmentsWithValue(uint, uint, uint) ([]Assessments, error) + GetAssessmentsWithValue(userID, assessment, size uint) ([]Assessments, error) GetUserCompletedAssesments(userID uint) (count int, err error) } @@ -52,6 +52,7 @@ type IDeployment interface { ListDeployments(userID, limit, offset uint) ([]Deployment, error) CreateDeployment(dt *Deployment) error GetDeploymentBy(opHash string) (*Deployment, error) + GetDeploymentsByAddressNetwork(address, network string) ([]Deployment, error) UpdateDeployment(dt *Deployment) error CountDeployments(userID uint) (int64, error) } @@ -61,16 +62,16 @@ type ISubscription interface { GetSubscription(userID uint, address, network string) (Subscription, error) GetSubscriptions(address, network string) ([]Subscription, error) ListSubscriptions(userID uint) ([]Subscription, error) - UpsertSubscription(*Subscription) error - DeleteSubscription(*Subscription) error + UpsertSubscription(s *Subscription) error + DeleteSubscription(s *Subscription) error GetSubscriptionsCount(address, network string) (int, error) } // IUser - type IUser interface { - GetOrCreateUser(*User, string) error - GetUser(uint) (*User, error) - UpdateUserMarkReadAt(uint, int64) error + GetOrCreateUser(u *User, token string) error + GetUser(userID uint) (*User, error) + UpdateUserMarkReadAt(userID uint, ts int64) error } // IVerification - diff --git a/internal/database/deployment.go b/internal/database/deployment.go index 3da71327a..80eaa634c 100644 --- a/internal/database/deployment.go +++ b/internal/database/deployment.go @@ -27,13 +27,10 @@ func (d *db) ListDeployments(userID, limit, offset uint) ([]Deployment, error) { req := d.Scopes( userIDScope(userID), pagination(limit, offset), - createdAtDesc) + createdAtDesc, + ) - if err := req.Find(&deployments).Error; err != nil { - return nil, err - } - - return deployments, nil + return deployments, req.Find(&deployments).Error } // CreateDeployment - @@ -47,6 +44,18 @@ func (d *db) GetDeploymentBy(opHash string) (*Deployment, error) { return dt, d.Raw("SELECT * FROM deployments WHERE operation_hash = ?", opHash).Scan(dt).Error } +// GetDeploymentsByAddressNetwork - +func (d *db) GetDeploymentsByAddressNetwork(address, network string) ([]Deployment, error) { + var deployments []Deployment + + req := d.Scopes( + addressScope(address), + networkScope(network), + ) + + return deployments, req.Find(&deployments).Error +} + // UpdateDeployment - func (d *db) UpdateDeployment(dt *Deployment) error { return d.Save(dt).Error diff --git a/internal/metrics/contract.go b/internal/metrics/contract.go index 2d2422231..733398c1d 100644 --- a/internal/metrics/contract.go +++ b/internal/metrics/contract.go @@ -3,7 +3,6 @@ package metrics import ( "github.com/baking-bad/bcdhub/internal/elastic" "github.com/baking-bad/bcdhub/internal/helpers" - "github.com/jinzhu/gorm" "github.com/baking-bad/bcdhub/internal/classification/functions" clmetrics "github.com/baking-bad/bcdhub/internal/classification/metrics" @@ -83,23 +82,3 @@ func compare(a, b models.Contract) bool { // log.Printf("%s -> %s [%d]", a.Address, b.Address, res) return res == 1 } - -// SetContractVerification - -func (h *Handler) SetContractVerification(c *models.Contract) error { - if c.Verified { - return nil - } - - v, err := h.DB.GetVerificationBy(c.Address, c.Network) - if err != nil { - if gorm.IsRecordNotFoundError(err) { - return nil - } - return err - } - - c.Verified = v.SourcePath != "" - c.VerificationSource = v.SourcePath - - return nil -} diff --git a/internal/metrics/operation.go b/internal/metrics/operation.go index b51a57672..2bf63dc98 100644 --- a/internal/metrics/operation.go +++ b/internal/metrics/operation.go @@ -5,13 +5,9 @@ import ( "log" "time" - "github.com/baking-bad/bcdhub/internal/compiler/compilation" "github.com/baking-bad/bcdhub/internal/contractparser/stringer" - "github.com/baking-bad/bcdhub/internal/database" - "github.com/baking-bad/bcdhub/internal/elastic" "github.com/baking-bad/bcdhub/internal/models" "github.com/getsentry/sentry-go" - "github.com/jinzhu/gorm" ) // SetOperationAliases - @@ -118,62 +114,3 @@ func initSentry(environment, dsn string) { log.Printf("Sentry initialization failed: %v\n", err) } } - -// SetOperationDeployment - -func (h *Handler) SetOperationDeployment(op *models.Operation) error { - d, err := h.DB.GetDeploymentBy(op.Hash) - if err != nil { - if gorm.IsRecordNotFoundError(err) { - return nil - } - return err - } - - d.Address = op.Destination - d.Network = op.Network - - if err := h.DB.UpdateDeployment(d); err != nil { - return err - } - - task, err := h.DB.GetCompilationTask(d.CompilationTaskID) - if err != nil { - return err - } - - var sourcePath string - - for _, r := range task.Results { - if r.Status == compilation.StatusSuccess { - sourcePath = r.AWSPath - break - } - } - - verification := database.Verification{ - UserID: task.UserID, - CompilationTaskID: d.CompilationTaskID, - Address: op.Destination, - Network: op.Network, - SourcePath: sourcePath, - } - - if err := h.DB.CreateVerification(&verification); err != nil { - return err - } - - contract := models.NewEmptyContract(op.Network, op.Destination) - if err := h.ES.GetByID(&contract); err != nil { - return err - } - - if !contract.Verified { - if err := h.SetContractVerification(&contract); err != nil { - return err - } - - return h.ES.UpdateFields(elastic.DocContracts, contract.GetID(), contract, "Verified", "VerificationSource") - } - - return nil -}