Implement block metrics (#252)

* block-metrics revision 1

* block-metrics revision 2

* Implement block-metrics

* Add new msgs

* Fix formatting err

* Be consistent with existing cluster env var

Co-authored-by: Dave Puchyr <[email protected]>

.travis.yml

@@ -37,6 +37,7 @@ env:
     - COMMIT=$(git log -1 --format='%H')
     - IMAGE_NAME="iov1/iovns:${BUILD_VERSION}"
     - FAUCET_IMAGE_NAME="iov1/iovns-faucet:${BUILD_VERSION}"
+    - BLOCk_METRICS_IMAGE_NAME="iov1/iovns-block-metrics:${BUILD_VERSION}"
 
 script:
   - set -eo pipefail
@@ -55,6 +56,7 @@ script:
     mkdir -p ${TRAVIS_BUILD_DIR}/build && tar -zxf $TRAVIS_BUILD_DIR/iovns-build-linux/build/out/iovns-${VERSION}-linux-amd64.tar.gz -C ${TRAVIS_BUILD_DIR}/build ;
     docker build --pull --tag ${IMAGE_NAME} . ;
     cd cmd/faucet && make build && docker build --pull --tag ${FAUCET_IMAGE_NAME} . ;
+    cd cmd/block-metrics && make build && docker build --pull --tag ${BLOCK_METRICS_IMAGE_NAME} . ;
     fi;
 
     if [[ $release_latest == "yes" ]]; then
@@ -64,15 +66,22 @@ script:
 
     docker tag  "iov1/iovns-faucet:${BUILD_VERSION}" "iov1/iovns-faucet:latest" ;
     docker push "iov1/iovns-faucet:latest";
+    docker tag  "iov1/iovns-block-metrics:${BUILD_VERSION}" "iov1/iovns-block-metrics:latest" ;
+    docker push "iov1/iovns-block-metrics:latest";
     docker logout;
     fi;
 
     if [[ $release_tag == "yes" ]]; then
     docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD";
     docker tag  "iov1/iovns:${BUILD_VERSION}" "iov1/iovns:$TRAVIS_TAG" ;
     docker push "iov1/iovns:$TRAVIS_TAG";
+
     docker tag  "iov1/iovns-faucet:${BUILD_VERSION}" "iov1/iovns-faucet:$TRAVIS_TAG" ;
     docker push "iov1/iovns-faucet:$TRAVIS_TAG";
+
+    docker tag  "iov1/iovns-block-metrics:${BUILD_VERSION}" "iov1/iovns-block-metrics:$TRAVIS_TAG" ;
+    docker push "iov1/iovns-block-metrics:$TRAVIS_TAG";
+
     docker logout;
     fi;
     fi;

CHANGELOG.md

@@ -5,6 +5,7 @@
 - FIX: AccountRenewalCountMax and DomainRenewalCountMax bumped at configuration update
 - FIX: fix cli tests
 - FIX: fix domain renewal
+- Implement block-metrics
 
 ## v0.4.4
 

cmd/block-metrics/.gitignore

@@ -0,0 +1,3 @@
+.database-data
+.envrc
+block-metrics

cmd/block-metrics/Dockerfile

@@ -0,0 +1,9 @@
+# Final image
+FROM alpine:edge
+
+WORKDIR /root
+
+# Copy over binaries from the build-env
+COPY /block-metrics /usr/bin/block-metrics
+
+CMD ["block-metrics"]

cmd/block-metrics/Makefile

@@ -0,0 +1,16 @@
+.PHONY: all install test
+
+# make sure we turn on go modules
+export GO111MODULE := on
+
+all: test install
+
+install:
+	go install .
+
+build:
+	GOARCH=amd64 CGO_ENABLED=0 GOOS=linux go build .
+
+test:
+	go vet -mod=readonly ./...
+	go test -mod=readonly -race ./...

cmd/block-metrics/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3.1'
+
+services:
+
+  db:
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: root
+    ports:
+      - "5432:5432"
+    volumes:
+      - .database-data:/var/lib/postgresql/data/
+

cmd/block-metrics/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/iov-one/iovns/cmd/block-metrics/pkg"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
+	"github.com/pkg/errors"
+)
+
+func main() {
+	conf := pkg.Configuration{
+		DBHost:          os.Getenv("POSTGRES_HOST"),
+		DBName:          os.Getenv("POSTGRES_DB"),
+		DBUser:          os.Getenv("POSTGRES_USER"),
+		DBPass:          os.Getenv("POSTGRES_PASSWORD"),
+		DBSSL:           os.Getenv("POSTGRES_SSL_ENABLE"),
+		TendermintWsURI: os.Getenv("TENDERMINT_WS_URI"),
+		Hrp:             os.Getenv("HRP"),
+	}
+
+	if err := run(conf); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(conf pkg.Configuration) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dbUri := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", conf.DBUser, conf.DBPass,
+		conf.DBHost, conf.DBName, conf.DBSSL)
+	db, err := sql.Open("postgres", dbUri)
+	if err != nil {
+		return fmt.Errorf("cannot connect to postgres: %s", err)
+	}
+	defer db.Close()
+
+	if err := pkg.EnsureSchema(db); err != nil {
+		return fmt.Errorf("ensure schema: %s", err)
+	}
+
+	st := pkg.NewStore(db)
+
+	tmc, err := pkg.DialTendermint(conf.TendermintWsURI)
+	if err != nil {
+		return errors.Wrap(err, "dial tendermint")
+	}
+	defer tmc.Close()
+
+	inserted, err := pkg.Sync(ctx, tmc, st, conf.Hrp)
+	if err != nil {
+		return errors.Wrap(err, "sync")
+	}
+
+	fmt.Println("inserted:", inserted)
+
+	return nil
+}

cmd/block-metrics/pkg/codec.go

@@ -0,0 +1,20 @@
+package pkg
+
+import (
+	"github.com/cosmos/cosmos-sdk/codec"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/iov-one/iovns/app"
+)
+
+var ModuleCdc *codec.Codec
+
+func init() {
+	ModuleCdc = app.MakeCodec()
+	config := sdk.GetConfig()
+	config.SetCoinType(app.CoinType)
+	config.SetFullFundraiserPath(app.FullFundraiserPath)
+	config.SetBech32PrefixForAccount(app.Bech32PrefixAccAddr, app.Bech32PrefixAccPub)
+	config.SetBech32PrefixForValidator(app.Bech32PrefixValAddr, app.Bech32PrefixValPub)
+	config.SetBech32PrefixForConsensusNode(app.Bech32PrefixConsAddr, app.Bech32PrefixConsPub)
+	config.Seal()
+}

cmd/block-metrics/pkg/config.go

@@ -0,0 +1,14 @@
+package pkg
+
+type Configuration struct {
+	DBHost string
+	DBUser string
+	DBPass string
+	DBName string
+	DBSSL  string
+	// Tendermint websocket URI
+	TendermintWsURI string
+	// Derivation path: "tiov" or "iov"
+	Hrp   string
+	Denom string
+}

cmd/block-metrics/pkg/errors.go

@@ -0,0 +1,48 @@
+package pkg
+
+import (
+	"database/sql"
+
+	"github.com/cosmos/cosmos-sdk/types/errors"
+	"github.com/lib/pq"
+)
+
+const ModuleName = "block-metrics"
+
+var (
+	ErrNotImplemented = errors.Register(ModuleName, 1, "not implemented")
+	ErrFailedResponse = errors.Register(ModuleName, 2, "failed response")
+	ErrConflict       = errors.Register(ModuleName, 3, "conflict")
+	ErrLimit          = errors.Register(ModuleName, 4, "limit")
+	ErrNotFound       = errors.Register(ModuleName, 5, "not found")
+	ErrDenom          = errors.Register(ModuleName, 6, "denomination not supported")
+)
+
+func wrapPgErr(err error, msg string) error {
+	if err == nil {
+		return nil
+	}
+	return errors.Wrap(castPgErr(err), msg)
+}
+
+func castPgErr(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	if err == sql.ErrNoRows {
+		return ErrNotFound
+	}
+
+	if e, ok := err.(*pq.Error); ok {
+		switch prefix := e.Code[:2]; prefix {
+		case "20":
+			return errors.Wrap(ErrNotFound, e.Message)
+		case "23":
+			return errors.Wrap(ErrConflict, e.Message)
+		}
+		err = errors.Wrap(err, string(e.Code))
+	}
+
+	return err
+}

cmd/block-metrics/pkg/json.go

@@ -0,0 +1,47 @@
+package pkg
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+// Tentermint is using hex encoded binary data. Provide a type that will do the
+// conversion as a part of JSON unmarshaling.
+type hexstring []byte
+
+func (h *hexstring) UnmarshalJSON(raw []byte) error {
+	var s string
+	if err := json.Unmarshal(raw, &s); err != nil {
+		return errors.Wrap(err, "invalid JSON string")
+	}
+	b, err := hex.DecodeString(s)
+	if err != nil {
+		return errors.Wrap(err, "hex decode")
+	}
+	*h = b
+	return nil
+}
+
+// Tentermint is using strings where a number is expected. Provide a type that
+// will do the conversion as a part of JSON unmarshaling.
+type sint64 int64
+
+func (i sint64) Int64() int64 {
+	return int64(i)
+}
+
+func (i *sint64) UnmarshalJSON(raw []byte) error {
+	var s string
+	if err := json.Unmarshal(raw, &s); err != nil {
+		return errors.Wrap(err, "invalid JSON string")
+	}
+	n, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return errors.Wrap(err, "invalid number")
+	}
+	*i = sint64(n)
+	return nil
+}

cmd/block-metrics/pkg/schema.go

@@ -0,0 +1,101 @@
+package pkg
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+)
+
+func EnsureSchema(pg *sql.DB) error {
+	tx, err := pg.Begin()
+	if err != nil {
+		return fmt.Errorf("transaction begin: %s", err)
+	}
+
+	for _, query := range strings.Split(schema, "\n---\n") {
+		query = strings.TrimSpace(query)
+
+		if _, err := tx.Exec(query); err != nil {
+			return &QueryError{Query: query, Err: err}
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("transaction commit: %s", err)
+	}
+
+	_ = tx.Rollback()
+
+	return nil
+}
+
+var schema = `
+CREATE TABLE IF NOT EXISTS blocks (
+	block_height BIGINT NOT NULL PRIMARY KEY,
+	block_hash TEXT NOT NULL UNIQUE,
+	block_time TIMESTAMPTZ NOT NULL,
+	fee_frac BIGINT NOT NULL
+);
+
+---
+
+CREATE TABLE IF NOT EXISTS transactions (
+	id BIGSERIAL PRIMARY KEY,
+	transaction_hash TEXT NOT NULL UNIQUE,
+	block_id BIGINT NOT NULL REFERENCES blocks(block_height),
+	signatures BYTEA ARRAY,
+	fee JSONB,
+	memo text
+);
+
+---
+CREATE TABLE IF NOT EXISTS domains (
+	id BIGSERIAL PRIMARY KEY,
+	name TEXT,
+	admin BYTEA NOT NULL,
+	type TEXT NOT NULL,
+	broker BYTEA,
+	fee_payer_addr BYTEA,
+	deleted_at TIMESTAMP
+);
+
+---
+CREATE TABLE IF NOT EXISTS accounts (
+	id BIGSERIAL PRIMARY KEY,
+	domain_id BIGINT NOT NULL REFERENCES domains(id),
+	domain TEXT NOT NULL,
+	name TEXT,
+	owner BYTEA NOT NULL,
+	registerer BYTEA,
+	broker BYTEA,
+	metadata_uri TEXT,
+	fee_payer_addr BYTEA,
+	deleted_at TIMESTAMP
+);
+
+---
+CREATE TABLE IF NOT EXISTS resources (
+	id BIGSERIAL PRIMARY KEY,
+	account_id BIGINT REFERENCES accounts(id),
+	uri TEXT,
+	resource TEXT
+);
+
+---
+CREATE TABLE IF NOT EXISTS account_certificates (
+	id BIGSERIAL PRIMARY KEY,
+	account_id BIGINT REFERENCES accounts(id),
+	certificate BYTEA,
+	deleted_at TIMESTAMP
+);
+`
+
+type QueryError struct {
+	Query string
+	Args  []interface{}
+	Err   error
+}
+
+func (e *QueryError) Error() string {
+	return fmt.Sprintf("query error: %s\n%q", e.Err, e.Query)
+}