Skip to content
This repository has been archived by the owner on Sep 28, 2021. It is now read-only.

Commit

Permalink
update commands
Browse files Browse the repository at this point in the history
  • Loading branch information
i-norden committed Aug 24, 2020
1 parent 5c5906c commit 7986e71
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 144 deletions.
71 changes: 71 additions & 0 deletions cmd/backfill.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
"fmt"

"github.com/spf13/viper"

"github.com/spf13/cobra"
)

// backfillCmd represents the backfill command
var backfillCmd = &cobra.Command{
Use: "backfill",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("backfill called")
},
}

func init() {
rootCmd.AddCommand(backfillCmd)

// flags for all config variables
backfillCmd.PersistentFlags().Int("backfill-frequency", 0, "how often (in seconds) the backfill process checks for gaps")
backfillCmd.PersistentFlags().Int("backfill-batch-size", 0, "data fetching batch size")
backfillCmd.PersistentFlags().Int("backfill-workers", 0, "how many goroutines to fetch data concurrently")
backfillCmd.PersistentFlags().Int("backfill-validation-level", 0, "backfill will rebackfill any data below this level")
backfillCmd.PersistentFlags().Int("backfill-timeout", 0, "timeout used for backfill http requests")

backfillCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
backfillCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
backfillCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
backfillCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
backfillCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
backfillCmd.PersistentFlags().String("eth-chain-id", "", "eth chain id")

// and their bindings
viper.BindPFlag("backfill.frequency", backfillCmd.PersistentFlags().Lookup("backfill-frequency"))
viper.BindPFlag("backfill.batchSize", backfillCmd.PersistentFlags().Lookup("backfill-batch-size"))
viper.BindPFlag("backfill.workers", backfillCmd.PersistentFlags().Lookup("backfill-workers"))
viper.BindPFlag("backfill.validationLevel", backfillCmd.PersistentFlags().Lookup("backfill-validation-level"))
viper.BindPFlag("backfill.timeout", backfillCmd.PersistentFlags().Lookup("backfill-timeout"))

viper.BindPFlag("ethereum.httpPath", backfillCmd.PersistentFlags().Lookup("eth-http-path"))
viper.BindPFlag("ethereum.nodeID", backfillCmd.PersistentFlags().Lookup("eth-node-id"))
viper.BindPFlag("ethereum.clientName", backfillCmd.PersistentFlags().Lookup("eth-client-name"))
viper.BindPFlag("ethereum.genesisBlock", backfillCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", backfillCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("ethereum.chainID", backfillCmd.PersistentFlags().Lookup("eth-chain-id"))
}
28 changes: 6 additions & 22 deletions cmd/resync.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,62 +49,46 @@ func rsyncCmdCommand() {
logWithCommand.Fatal(err)
}
logWithCommand.Info("starting up resync process")
if err := rService.Resync(); err != nil {
if err := rService.Sync(); err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("%s %s resync finished", rConfig.Chain.String(), rConfig.ResyncType.String())
logWithCommand.Infof("ethereum %s resync finished", rConfig.ResyncType.String())
}

func init() {
rootCmd.AddCommand(resyncCmd)

// flags
resyncCmd.PersistentFlags().String("resync-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
resyncCmd.PersistentFlags().String("resync-type", "", "which type of data to resync")
resyncCmd.PersistentFlags().Int("resync-start", 0, "block height to start resync")
resyncCmd.PersistentFlags().Int("resync-stop", 0, "block height to stop resync")
resyncCmd.PersistentFlags().Int("resync-batch-size", 0, "data fetching batch size")
resyncCmd.PersistentFlags().Int("resync-batch-number", 0, "how many goroutines to fetch data concurrently")
resyncCmd.PersistentFlags().Int("resync-workers", 0, "how many goroutines to fetch data concurrently")
resyncCmd.PersistentFlags().Bool("resync-clear-old-cache", false, "if true, clear out old data of the provided type within the resync range before resyncing")
resyncCmd.PersistentFlags().Bool("resync-reset-validation", false, "if true, reset times_validated to 0")
resyncCmd.PersistentFlags().Int("resync-timeout", 15, "timeout used for resync http requests")

resyncCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
resyncCmd.PersistentFlags().String("btc-password", "", "password for btc node")
resyncCmd.PersistentFlags().String("btc-username", "", "username for btc node")
resyncCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
resyncCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
resyncCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
resyncCmd.PersistentFlags().String("btc-network-id", "", "btc network id")

resyncCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
resyncCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
resyncCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
resyncCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
resyncCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
resyncCmd.PersistentFlags().String("eth-chain-id", "", "eth chain id")

// and their bindings
viper.BindPFlag("resync.chain", resyncCmd.PersistentFlags().Lookup("resync-chain"))
viper.BindPFlag("resync.type", resyncCmd.PersistentFlags().Lookup("resync-type"))
viper.BindPFlag("resync.start", resyncCmd.PersistentFlags().Lookup("resync-start"))
viper.BindPFlag("resync.stop", resyncCmd.PersistentFlags().Lookup("resync-stop"))
viper.BindPFlag("resync.batchSize", resyncCmd.PersistentFlags().Lookup("resync-batch-size"))
viper.BindPFlag("resync.batchNumber", resyncCmd.PersistentFlags().Lookup("resync-batch-number"))
viper.BindPFlag("resync.workers", resyncCmd.PersistentFlags().Lookup("resync-workers"))
viper.BindPFlag("resync.clearOldCache", resyncCmd.PersistentFlags().Lookup("resync-clear-old-cache"))
viper.BindPFlag("resync.resetValidation", resyncCmd.PersistentFlags().Lookup("resync-reset-validation"))
viper.BindPFlag("resync.timeout", resyncCmd.PersistentFlags().Lookup("resync-timeout"))

viper.BindPFlag("bitcoin.httpPath", resyncCmd.PersistentFlags().Lookup("btc-http-path"))
viper.BindPFlag("bitcoin.pass", resyncCmd.PersistentFlags().Lookup("btc-password"))
viper.BindPFlag("bitcoin.user", resyncCmd.PersistentFlags().Lookup("btc-username"))
viper.BindPFlag("bitcoin.nodeID", resyncCmd.PersistentFlags().Lookup("btc-node-id"))
viper.BindPFlag("bitcoin.clientName", resyncCmd.PersistentFlags().Lookup("btc-client-name"))
viper.BindPFlag("bitcoin.genesisBlock", resyncCmd.PersistentFlags().Lookup("btc-genesis-block"))
viper.BindPFlag("bitcoin.networkID", resyncCmd.PersistentFlags().Lookup("btc-network-id"))

viper.BindPFlag("ethereum.httpPath", resyncCmd.PersistentFlags().Lookup("eth-http-path"))
viper.BindPFlag("ethereum.nodeID", resyncCmd.PersistentFlags().Lookup("eth-node-id"))
viper.BindPFlag("ethereum.clientName", resyncCmd.PersistentFlags().Lookup("eth-client-name"))
viper.BindPFlag("ethereum.genesisBlock", resyncCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", resyncCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("ethereum.chainID", resyncCmd.PersistentFlags().Lookup("eth-chain-id"))
}
153 changes: 31 additions & 122 deletions cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,17 @@ import (
"os/signal"
s "sync"

"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"

h "github.com/vulcanize/ipfs-blockchain-watcher/pkg/historical"
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/shared"
w "github.com/vulcanize/ipfs-blockchain-watcher/pkg/watch"
v "github.com/vulcanize/ipfs-blockchain-watcher/version"
)

// watchCmd represents the watch command
var watchCmd = &cobra.Command{
Use: "watch",
// syncCmd represents the sync command
var syncCmd = &cobra.Command{
Use: "sync",
Short: "sync chain data into PG-IPFS",
Long: `This command configures a VulcanizeDB ipfs-blockchain-watcher.
Expand All @@ -49,146 +46,58 @@ and fill in gaps in the data
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *log.WithField("SubCommand", subCommand)
watch()
sync()
},
}

func watch() {
func sync() {
logWithCommand.Infof("running ipfs-blockchain-watcher version: %s", v.VersionWithMeta)

var forwardPayloadChan chan shared.ConvertedData
wg := new(s.WaitGroup)
logWithCommand.Debug("loading watcher configuration variables")
watcherConfig, err := w.NewConfig()
logWithCommand.Debug("loading configuration variables")
syncerConfig, err := w.NewConfig()
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("watcher config: %+v", watcherConfig)
logWithCommand.Debug("initializing new watcher service")
watcher, err := w.NewWatcher(watcherConfig)
logWithCommand.Infof("config: %+v", syncerConfig)
logWithCommand.Debug("initializing new indexing service")
syncer, err := w.NewIndexerService(syncerConfig)
if err != nil {
logWithCommand.Fatal(err)
}

if watcherConfig.Serve {
logWithCommand.Info("starting up watcher servers")
forwardPayloadChan = make(chan shared.ConvertedData, w.PayloadChanBufferSize)
watcher.Serve(wg, forwardPayloadChan)
if err := startServers(watcher, watcherConfig); err != nil {
logWithCommand.Fatal(err)
}
}

if watcherConfig.Sync {
logWithCommand.Info("starting up watcher sync process")
if err := watcher.Sync(wg, forwardPayloadChan); err != nil {
logWithCommand.Fatal(err)
}
}

var backFiller h.BackFillInterface
if watcherConfig.Historical {
historicalConfig, err := h.NewConfig()
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Debug("initializing new historical backfill service")
backFiller, err = h.NewBackFillService(historicalConfig, forwardPayloadChan)
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Info("starting up watcher backfill process")
backFiller.BackFill(wg)
logWithCommand.Info("starting up sync process")
if err := syncer.Sync(wg); err != nil {
logWithCommand.Fatal(err)
}

shutdown := make(chan os.Signal)
signal.Notify(shutdown, os.Interrupt)
<-shutdown
if watcherConfig.Historical {
backFiller.Stop()
}
watcher.Stop()
syncer.Stop()
wg.Wait()
}

func startServers(watcher w.Watcher, settings *w.Config) error {
logWithCommand.Debug("starting up IPC server")
_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, watcher.APIs())
if err != nil {
return err
}
logWithCommand.Debug("starting up WS server")
_, _, err = rpc.StartWSEndpoint(settings.WSEndpoint, watcher.APIs(), []string{"vdb"}, nil, true)
if err != nil {
return err
}
logWithCommand.Debug("starting up HTTP server")
_, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, watcher.APIs(), []string{settings.Chain.API()}, nil, nil, rpc.HTTPTimeouts{})
return err
}

func init() {
rootCmd.AddCommand(watchCmd)
rootCmd.AddCommand(syncCmd)

// flags for all config variables
watchCmd.PersistentFlags().String("watcher-chain", "", "which chain to support, options are currently Ethereum or Bitcoin.")
watchCmd.PersistentFlags().Bool("watcher-server", false, "turn vdb server on or off")
watchCmd.PersistentFlags().String("watcher-ws-path", "", "vdb server ws path")
watchCmd.PersistentFlags().String("watcher-http-path", "", "vdb server http path")
watchCmd.PersistentFlags().String("watcher-ipc-path", "", "vdb server ipc path")
watchCmd.PersistentFlags().Bool("watcher-sync", false, "turn vdb sync on or off")
watchCmd.PersistentFlags().Int("watcher-workers", 0, "how many worker goroutines to publish and index data")
watchCmd.PersistentFlags().Bool("watcher-back-fill", false, "turn vdb backfill on or off")
watchCmd.PersistentFlags().Int("watcher-frequency", 0, "how often (in seconds) the backfill process checks for gaps")
watchCmd.PersistentFlags().Int("watcher-batch-size", 0, "data fetching batch size")
watchCmd.PersistentFlags().Int("watcher-batch-number", 0, "how many goroutines to fetch data concurrently")
watchCmd.PersistentFlags().Int("watcher-validation-level", 0, "backfill will resync any data below this level")
watchCmd.PersistentFlags().Int("watcher-timeout", 0, "timeout used for backfill http requests")

watchCmd.PersistentFlags().String("btc-ws-path", "", "ws url for bitcoin node")
watchCmd.PersistentFlags().String("btc-http-path", "", "http url for bitcoin node")
watchCmd.PersistentFlags().String("btc-password", "", "password for btc node")
watchCmd.PersistentFlags().String("btc-username", "", "username for btc node")
watchCmd.PersistentFlags().String("btc-node-id", "", "btc node id")
watchCmd.PersistentFlags().String("btc-client-name", "", "btc client name")
watchCmd.PersistentFlags().String("btc-genesis-block", "", "btc genesis block hash")
watchCmd.PersistentFlags().String("btc-network-id", "", "btc network id")

watchCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
watchCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
watchCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
watchCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
watchCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
watchCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
syncCmd.PersistentFlags().Int("sync-workers", 0, "how many worker goroutines to publish and index data")

syncCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
syncCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
syncCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
syncCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
syncCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
syncCmd.PersistentFlags().String("eth-chain-id", "", "eth chain id")

// and their bindings
viper.BindPFlag("watcher.chain", watchCmd.PersistentFlags().Lookup("watcher-chain"))
viper.BindPFlag("watcher.server", watchCmd.PersistentFlags().Lookup("watcher-server"))
viper.BindPFlag("watcher.wsPath", watchCmd.PersistentFlags().Lookup("watcher-ws-path"))
viper.BindPFlag("watcher.httpPath", watchCmd.PersistentFlags().Lookup("watcher-http-path"))
viper.BindPFlag("watcher.ipcPath", watchCmd.PersistentFlags().Lookup("watcher-ipc-path"))
viper.BindPFlag("watcher.sync", watchCmd.PersistentFlags().Lookup("watcher-sync"))
viper.BindPFlag("watcher.workers", watchCmd.PersistentFlags().Lookup("watcher-workers"))
viper.BindPFlag("watcher.backFill", watchCmd.PersistentFlags().Lookup("watcher-back-fill"))
viper.BindPFlag("watcher.frequency", watchCmd.PersistentFlags().Lookup("watcher-frequency"))
viper.BindPFlag("watcher.batchSize", watchCmd.PersistentFlags().Lookup("watcher-batch-size"))
viper.BindPFlag("watcher.batchNumber", watchCmd.PersistentFlags().Lookup("watcher-batch-number"))
viper.BindPFlag("watcher.validationLevel", watchCmd.PersistentFlags().Lookup("watcher-validation-level"))
viper.BindPFlag("watcher.timeout", watchCmd.PersistentFlags().Lookup("watcher-timeout"))

viper.BindPFlag("bitcoin.wsPath", watchCmd.PersistentFlags().Lookup("btc-ws-path"))
viper.BindPFlag("bitcoin.httpPath", watchCmd.PersistentFlags().Lookup("btc-http-path"))
viper.BindPFlag("bitcoin.pass", watchCmd.PersistentFlags().Lookup("btc-password"))
viper.BindPFlag("bitcoin.user", watchCmd.PersistentFlags().Lookup("btc-username"))
viper.BindPFlag("bitcoin.nodeID", watchCmd.PersistentFlags().Lookup("btc-node-id"))
viper.BindPFlag("bitcoin.clientName", watchCmd.PersistentFlags().Lookup("btc-client-name"))
viper.BindPFlag("bitcoin.genesisBlock", watchCmd.PersistentFlags().Lookup("btc-genesis-block"))
viper.BindPFlag("bitcoin.networkID", watchCmd.PersistentFlags().Lookup("btc-network-id"))

viper.BindPFlag("ethereum.wsPath", watchCmd.PersistentFlags().Lookup("eth-ws-path"))
viper.BindPFlag("ethereum.httpPath", watchCmd.PersistentFlags().Lookup("eth-http-path"))
viper.BindPFlag("ethereum.nodeID", watchCmd.PersistentFlags().Lookup("eth-node-id"))
viper.BindPFlag("ethereum.clientName", watchCmd.PersistentFlags().Lookup("eth-client-name"))
viper.BindPFlag("ethereum.genesisBlock", watchCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", watchCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("sync.workers", syncCmd.PersistentFlags().Lookup("sync-workers"))

viper.BindPFlag("ethereum.wsPath", syncCmd.PersistentFlags().Lookup("eth-ws-path"))
viper.BindPFlag("ethereum.nodeID", syncCmd.PersistentFlags().Lookup("eth-node-id"))
viper.BindPFlag("ethereum.clientName", syncCmd.PersistentFlags().Lookup("eth-client-name"))
viper.BindPFlag("ethereum.genesisBlock", syncCmd.PersistentFlags().Lookup("eth-genesis-block"))
viper.BindPFlag("ethereum.networkID", syncCmd.PersistentFlags().Lookup("eth-network-id"))
viper.BindPFlag("ethereum.chainID", syncCmd.PersistentFlags().Lookup("eth-chain-id"))
}

0 comments on commit 7986e71

Please sign in to comment.