bidcollect: fn with uid

cmd/service/bidcollect.go

@@ -9,6 +9,7 @@ import (
 	"github.com/flashbots/relayscan/services/bidcollect"
 	"github.com/flashbots/relayscan/services/bidcollect/website"
 	"github.com/flashbots/relayscan/vars"
+	"github.com/lithammer/shortuuid"
 	"github.com/spf13/cobra"
 )
 
@@ -19,7 +20,8 @@ var (
 	useAllRelays            bool
 
 	outDir    string
-	outputTSV bool // by default: CSV, but can be changed to TSV with this setting
+	outputTSV bool   // by default: CSV, but can be changed to TSV with this setting
+	uid       string // used in output filenames, to avoid collissions between multiple collector instances
 
 	runDevServerOnly    bool // used to play with file listing website
 	devServerListenAddr string
@@ -42,6 +44,9 @@ func init() {
 	bidCollectCmd.Flags().StringVar(&outDir, "out", "csv", "output directory for CSV/TSV")
 	bidCollectCmd.Flags().BoolVar(&outputTSV, "out-tsv", false, "output as TSV (instead of CSV)")
 
+	// utils
+	bidCollectCmd.Flags().StringVar(&uid, "uid", "", "unique identifier for output files (to avoid collisions)")
+
 	// for dev purposes
 	bidCollectCmd.Flags().BoolVar(&runDevServerOnly, "devserver", false, "only run devserver to play with file listing website")
 	bidCollectCmd.Flags().StringVar(&devServerListenAddr, "devserver-addr", "localhost:8095", "listen address for devserver")
@@ -68,7 +73,11 @@ var bidCollectCmd = &cobra.Command{
 			return
 		}
 
-		log.Infof("Bidcollect %s starting ...", vars.Version)
+		if uid == "" {
+			uid = shortuuid.New()[:6]
+		}
+
+		log.WithField("uid", uid).Infof("Bidcollect %s starting ...", vars.Version)
 
 		// Prepare relays
 		relays := []common.RelayEntry{
@@ -86,6 +95,7 @@ var bidCollectCmd = &cobra.Command{
 
 		opts := bidcollect.BidCollectorOpts{
 			Log:                     log,
+			UID:                     uid,
 			Relays:                  relays,
 			CollectUltrasoundStream: collectUltrasoundStream,
 			CollectGetHeader:        collectGetHeader,

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.9
+	github.com/lithammer/shortuuid v3.0.0+incompatible
 	github.com/metachris/flashbotsrpc v0.5.0
 	github.com/metachris/go-ethutils v0.5.3
 	github.com/olekukonko/tablewriter v0.0.5

go.sum

@@ -470,6 +470,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w=
+github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

scripts/bidcollect/bids-combine-and-upload-yesterday.sh

@@ -27,4 +27,4 @@ source .env.prod
 ./scripts/bidcollect/bids-combine-and-upload.sh "/mnt/data/relayscan-bids/$d/"
 
 # update website
-make update-bids-website
+./relayscan service bidcollect --build-website --build-website-upload

services/bidcollect/bid-processor.go

@@ -20,6 +20,7 @@ import (
 
 type BidProcessorOpts struct {
 	Log       *logrus.Entry
+	UID       string
 	OutDir    string
 	OutputTSV bool
 }
@@ -202,7 +203,7 @@ func (c *BidProcessor) getFilename(prefix string, timestamp int64) string {
 	if prefix != "" {
 		prefix += "_"
 	}
-	return fmt.Sprintf("%s%s.%s", prefix, t.Format("2006-01-02_15-04"), c.csvFileEnding)
+	return fmt.Sprintf("%s%s_%s.%s", prefix, t.Format("2006-01-02_15-04"), c.opts.UID, c.csvFileEnding)
 }
 
 func (c *BidProcessor) housekeeping() {

services/bidcollect/bidcollector.go

@@ -8,6 +8,7 @@ import (
 
 type BidCollectorOpts struct {
 	Log *logrus.Entry
+	UID string
 
 	CollectUltrasoundStream bool
 	CollectGetHeader        bool
@@ -49,6 +50,7 @@ func NewBidCollector(opts *BidCollectorOpts) *BidCollector {
 	// output
 	c.processor = NewBidProcessor(&BidProcessorOpts{
 		Log:       opts.Log,
+		UID:       opts.UID,
 		OutDir:    opts.OutDir,
 		OutputTSV: opts.OutputTSV,
 	})