Periodic testnets (#958)

* Testnets config and drop database

* Drop database on new chain detection

* Enable runtime vars in bcd, add necessary scripts for resetting periodic testnets

* Add rollupnet

* Teztnets: fix api bind

* Bump gui ver

* Restart periodic networks

* Fix: workflow

* Linter configuration

* go.nod

* Fix: linter

* Fix: linter

* Fix: indexer crreations

* Add rollupnet teznets source

---------

Co-authored-by: Michael Zaikin <[email protected]>
Co-authored-by: Michael Zaikin <[email protected]>

.github/workflows/build.yml

@@ -102,7 +102,7 @@ jobs:
           context: .
           file: build/sandbox/Dockerfile
           build-args: |
-            TAG=4.3.0
+            TAG=feat/teztnets
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max

.github/workflows/tests_master.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
@@ -24,7 +24,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18.x
+          go-version: 1.19.x
       - name: checkout code
         uses: actions/checkout@v2
       - uses: actions/cache@v2

.github/workflows/tests_pr.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
@@ -25,7 +25,7 @@ jobs:
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.18.x
+          go-version: 1.19.x
       - name: checkout code
         uses: actions/checkout@v2
       - uses: actions/cache@v2

.golangci.yml

@@ -11,3 +11,7 @@ linters:
     - ineffassign
     - containedctx
     - tenv
+
+run:
+  go: '1.19'
+  timeout: 3m

Makefile

@@ -59,7 +59,7 @@ test:
 	go test ./...
 
 lint:
-	golangci-lint run --go=1.18
+	golangci-lint run
 
 test-api:
 	# to install newman:

build/api/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.18-alpine as builder
+FROM golang:1.19-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on
@@ -33,7 +33,8 @@ RUN cd nginx && go build -a -installsuffix cgo -o /go/bin/seo .
 # ---------------------------------------------------------------------
 #  The second stage container, for running the application
 # ---------------------------------------------------------------------
-FROM scratch
+FROM alpine:3.16
+RUN apk add curl jq
 
 ENV PATH="/go/bin/:$PATH"
 
@@ -47,4 +48,7 @@ COPY --from=builder /go/bin/bcdctl /go/bin/bcdctl
 COPY --from=builder /go/bin/migration /go/bin/migration
 COPY --from=builder /go/bin/seo /go/bin/seo
 
+COPY build/api/entrypoint.sh build/api/healthcheck.sh /
+RUN chmod +x /entrypoint.sh /healthcheck.sh
+
 ENTRYPOINT ["/go/bin/api"]

build/api/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+curl https://teztnets.xyz/teztnets.json > /teztnets.json
+mondaynet_rpc=$(cat /teztnets.json | jq '. | to_entries | map(select(.key | startswith("monday"))) | map(.value.rpc_url)[0]')
+dailynet_rpc=$(cat /teztnets.json | jq '. | to_entries | map(select(.key | startswith("daily"))) | map(.value.rpc_url)[0]')
+
+DAILYNET_RPC=$dailynet_rpc MONDAYNET_RPC=$mondaynet_rpc /go/bin/api

build/api/healthcheck.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+diff -s <(curl https://teztnets.xyz/teztnets.json) /teztnets.json

build/indexer/Dockerfile

@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.18-alpine as builder
+FROM golang:1.19-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on
@@ -25,12 +25,16 @@ RUN go build -a -installsuffix cgo -o /go/bin/indexer .
 # ---------------------------------------------------------------------
 #  The second stage container, for running the application
 # ---------------------------------------------------------------------
-FROM scratch
+FROM alpine:3.16
+RUN apk add curl jq
 
 WORKDIR /app/indexer
 
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /go/bin/indexer /go/bin/indexer
 COPY configs/*.yml /app/indexer/
 
+COPY build/indexer/entrypoint.sh build/indexer/healthcheck.sh /
+RUN chmod +x /entrypoint.sh /healthcheck.sh
+
 ENTRYPOINT ["/go/bin/indexer"]

build/indexer/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+curl https://teztnets.xyz/teztnets.json > /teztnets.json
+mondaynet_rpc=$(cat /teztnets.json | jq '. | to_entries | map(select(.key | startswith("monday"))) | map(.value.rpc_url)[0]')
+dailynet_rpc=$(cat /teztnets.json | jq '. | to_entries | map(select(.key | startswith("daily"))) | map(.value.rpc_url)[0]')
+
+DAILYNET_RPC=$dailynet_rpc MONDAYNET_RPC=$mondaynet_rpc /go/bin/indexer

build/indexer/healthcheck.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+diff -s <(curl https://teztnets.xyz/teztnets.json) /teztnets.json

build/sandbox/Dockerfile

@@ -12,5 +12,8 @@ RUN yarn build
 
 FROM nginx:latest AS release
 COPY build/sandbox/default.conf /etc/nginx/conf.d/default.conf
+COPY build/sandbox/entrypoint.sh /
+RUN chmod +x /entrypoint.sh
 WORKDIR /usr/share/nginx/html/
-COPY --from=build /bcd/dist ./
+COPY --from=build /bcd/dist ./
+ENTRYPOINT [ "/entrypoint.sh" ]

build/sandbox/entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+ROOT_DIR=/usr/share/nginx/html
+
+if [ -z "$BASE_URL" ]; then
+    BASE_URL=http://localhost
+fi
+if [ -z "$API_URL" ]; then
+    API_URL=http://localhost:14000/v1
+fi
+if [ -z "$IPFS_URL" ]; then
+    IPFS_URL=https://ipfs.io
+fi
+
+for file in $ROOT_DIR/js/*.js* $ROOT_DIR/index.html;
+do
+  sed -i 's|__BASE_URL__|'$BASE_URL'|g' $file
+  sed -i 's|__VUE_APP_API_URI__|'$API_URL'|g' $file
+  sed -i 's|__VUE_APP_IPFS_NODE__|'$IPFS_URL'|g' $file
+done
+
+nginx -g 'daemon off;'

build/sandbox/env.production

@@ -1,3 +1,4 @@
 NODE_ENV=production
-VUE_APP_API_URI=http://localhost:14000/v1
-VUE_APP_IPFS_NODE=https://ipfs.io
+BASE_URL=__BASE_URL__
+VUE_APP_API_URI=__VUE_APP_API_URI__
+VUE_APP_IPFS_NODE=__VUE_APP_IPFS_NODE__

cmd/api/main.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
@@ -10,6 +13,7 @@ import (
 	"github.com/baking-bad/bcdhub/internal/config"
 	"github.com/baking-bad/bcdhub/internal/helpers"
 	"github.com/baking-bad/bcdhub/internal/logger"
+	"github.com/baking-bad/bcdhub/internal/periodic"
 	"github.com/gin-contrib/cache"
 	"github.com/gin-contrib/cache/persistence"
 	"github.com/gin-contrib/cors"
@@ -22,6 +26,9 @@ type app struct {
 	Router   *gin.Engine
 	Contexts config.Contexts
 	Config   config.Config
+
+	cancel context.CancelFunc
+	worker *periodic.GeneralWorker
 }
 
 func newApp() *app {
@@ -36,19 +43,35 @@ func newApp() *app {
 		defer helpers.CatchPanicSentry()
 	}
 
-	api := &app{
-		Contexts: config.NewContexts(cfg, cfg.API.Networks,
-			config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize), cfg.API.Connections.Open, cfg.API.Connections.Idle, false),
-			config.WithRPC(cfg.RPC),
-			config.WithMempool(cfg.Services),
-			config.WithLoadErrorDescriptions(),
-			config.WithConfigCopy(cfg)),
-		Config: cfg,
+	app := new(app)
+	app.Config = cfg
+
+	if cfg.API.Periodic != nil {
+		worker, err := periodic.NewGeneralWorker(*cfg.API.Periodic, app.handleUrlChanged)
+		if err != nil {
+			panic(err)
+		}
+		app.worker = worker
+
+		ctx, cancel := context.WithCancel(context.Background())
+		app.cancel = cancel
+		app.worker.Start(ctx)
+
+		for len(app.worker.URLs()) == 0 {
+			time.Sleep(time.Second)
+		}
 	}
 
-	api.makeRouter()
+	app.Contexts = config.NewContexts(cfg, cfg.API.Networks,
+		config.WithStorage(cfg.Storage, cfg.API.ProjectName, int64(cfg.API.PageSize), cfg.API.Connections.Open, cfg.API.Connections.Idle, false),
+		config.WithRPC(cfg.RPC),
+		config.WithMempool(cfg.Services),
+		config.WithLoadErrorDescriptions(),
+		config.WithConfigCopy(cfg))
 
-	return api
+	app.makeRouter()
+
+	return app
 }
 
 func (api *app) makeRouter() {
@@ -194,15 +217,24 @@ func (api *app) makeRouter() {
 	api.Router = r
 }
 
-func (api *app) Close() {
-	api.Contexts.Close()
+// Close -
+func (api *app) Close() error {
+	api.cancel()
+
+	if err := api.Contexts.Close(); err != nil {
+		return err
+	}
+	return api.worker.Close()
 }
 
+// Run -
 func (api *app) Run() {
 	if err := api.Router.Run(api.Config.API.Bind); err != nil {
+		if errors.Is(err, http.ErrServerClosed) {
+			return
+		}
 		logger.Err(err)
 		helpers.CatchErrorSentry(err)
-		return
 	}
 }
 
@@ -246,3 +278,16 @@ func loggerFormat() gin.HandlerFunc {
 		)
 	})
 }
+
+func (api *app) handleUrlChanged(ctx context.Context, network, url string) error {
+	if value, ok := api.Config.RPC[network]; ok {
+		value.URI = url
+		api.Config.RPC[network] = value
+	}
+
+	if _, ok := api.Config.API.Frontend.RPC[network]; ok {
+		api.Config.API.Frontend.RPC[network] = url
+	}
+
+	return nil
+}

cmd/indexer/indexer/create.go

@@ -2,11 +2,11 @@ package indexer
 
 import (
 	"context"
+	"sync"
 
 	"github.com/baking-bad/bcdhub/internal/bcd/tezerrors"
 	"github.com/baking-bad/bcdhub/internal/config"
-	"github.com/baking-bad/bcdhub/internal/models/types"
-	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 // CreateIndexers -
@@ -15,18 +15,40 @@ func CreateIndexers(ctx context.Context, cfg config.Config) ([]Indexer, error) {
 		return nil, err
 	}
 
-	indexers := make([]Indexer, 0)
+	var (
+		mx       sync.Mutex
+		wg       sync.WaitGroup
+		indexers = make([]Indexer, 0)
+	)
+
 	for network, indexerCfg := range cfg.Indexer.Networks {
-		networkType := types.NewNetwork(network)
-		if networkType == types.Empty {
-			return nil, errors.Errorf("unknown network %s", network)
-		}
-
-		bi, err := NewBlockchainIndexer(ctx, cfg, networkType, indexerCfg)
-		if err != nil {
-			return nil, err
-		}
-		indexers = append(indexers, bi)
+		wg.Add(1)
+		go func(network string, indexerCfg config.IndexerConfig) {
+			defer wg.Done()
+
+			if indexerCfg.Periodic != nil {
+				periodicIndexer, err := NewPeriodicIndexer(ctx, network, cfg, indexerCfg)
+				if err != nil {
+					log.Err(err).Msg("NewPeriodicIndexer")
+					return
+				}
+				mx.Lock()
+				indexers = append(indexers, periodicIndexer)
+				mx.Unlock()
+			} else {
+				bi, err := NewBlockchainIndexer(ctx, cfg, network, indexerCfg)
+				if err != nil {
+					log.Err(err).Msg("NewBlockchainIndexer")
+					return
+				}
+				mx.Lock()
+				indexers = append(indexers, bi)
+				mx.Unlock()
+			}
+		}(network, indexerCfg)
 	}
+
+	wg.Wait()
+
 	return indexers, nil
 }