bidcollect: fn with uid

cmd/service/bidcollect.go

@@ -9,6 +9,7 @@ import (
 	"github.com/flashbots/relayscan/services/bidcollect"
 	"github.com/flashbots/relayscan/services/bidcollect/website"
 	"github.com/flashbots/relayscan/vars"
+	"github.com/lithammer/shortuuid"
 	"github.com/spf13/cobra"
 )
 
@@ -19,7 +20,8 @@ var (
 	useAllRelays            bool
 
 	outDir    string
-	outputTSV bool // by default: CSV, but can be changed to TSV with this setting
+	outputTSV bool   // by default: CSV, but can be changed to TSV with this setting
+	uid       string // used in output filenames, to avoid collissions between multiple collector instances
 
 	runDevServerOnly    bool // used to play with file listing website
 	devServerListenAddr string
@@ -42,6 +44,9 @@ func init() {
 	bidCollectCmd.Flags().StringVar(&outDir, "out", "csv", "output directory for CSV/TSV")
 	bidCollectCmd.Flags().BoolVar(&outputTSV, "out-tsv", false, "output as TSV (instead of CSV)")
 
+	// utils
+	bidCollectCmd.Flags().StringVar(&uid, "uid", "", "unique identifier for output files (to avoid collisions)")
+
 	// for dev purposes
 	bidCollectCmd.Flags().BoolVar(&runDevServerOnly, "devserver", false, "only run devserver to play with file listing website")
 	bidCollectCmd.Flags().StringVar(&devServerListenAddr, "devserver-addr", "localhost:8095", "listen address for devserver")
@@ -68,7 +73,11 @@ var bidCollectCmd = &cobra.Command{
 			return
 		}
 
-		log.Infof("Bidcollect %s starting ...", vars.Version)
+		if uid == "" {
+			uid = shortuuid.New()[:6]
+		}
+
+		log.WithField("uid", uid).Infof("Bidcollect %s starting ...", vars.Version)
 
 		// Prepare relays
 		relays := []common.RelayEntry{
@@ -86,6 +95,7 @@ var bidCollectCmd = &cobra.Command{
 
 		opts := bidcollect.BidCollectorOpts{
 			Log:                     log,
+			UID:                     uid,
 			Relays:                  relays,
 			CollectUltrasoundStream: collectUltrasoundStream,
 			CollectGetHeader:        collectGetHeader,

docs/2024-06_bidcollect.md

@@ -96,19 +96,23 @@ go run . service bidcollect --get-header --beacon-uri http://localhost:3500 --al
 Useful [clickhouse-local](https://clickhouse.com/docs/en/operations/utilities/clickhouse-local) queries:
 
 ```bash
-$ clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '2024-06-12_top.csv' GROUP BY source_type ORDER BY source_type;"
+# Set the CSV filename, for ease of reuse across queries
+$ fn=2024-06-12_top.csv
+
+# Count different source types in file
+$ clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '$fn' GROUP BY source_type ORDER BY source_type;"
 0       2929
 1       21249
 2       1057722
 
-# Get bids > 1 ETH for specific builders (CSV has 10M rows)
-$ time clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '2024-06-05_all.csv' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0xa01a00479f1fa442a8ebadb352be69091d07b0c0a733fae9166dae1b83179e326a968717da175c7363cd5a13e8580e8d', '0xa02a0054ea4ba422c88baccfdb1f43b2c805f01d1475335ea6647f69032da847a41c0e23796c6bed39b0ee11ab9772c6', '0xa03a000b0e3d1dc008f6075a1b1af24e6890bd674c26235ce95ac06e86f2bd3ccf4391df461b9e5d3ca654ef6b9e1ceb') FORMAT TabSeparatedWithNames;"
-count(value)    p50     p75     p90     max(value)
-1842    1789830446982354000     2279820737908906200     4041286254343376400     8216794401676997763
+# Count optimistic mode
+$ clickhouse local -q "SELECT optimistic_submission, COUNT(optimistic_submission) FROM '$fn' WHERE optimistic_submission IS NOT NULL GROUP BY optimistic_submission;"
+
+# Count bids with >1 ETH in value, by builder
+$ clickhouse local -q "SELECT builder_pubkey, count(builder_pubkey) as count, quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '$fn' WHERE value > 1000000000000000000 AND builder_pubkey != '' GROUP BY builder_pubkey ORDER BY count DESC FORMAT TabSeparatedWithNames;"
 
-real    0m2.202s
-user    0m17.320s
-sys     0m0.589s
+# Get bids > 1 ETH for specific builders
+$ clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '$fn' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0x...', '0x...', '0x...') FORMAT TabSeparatedWithNames;"
 ```
 
 ---

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.9
+	github.com/lithammer/shortuuid v3.0.0+incompatible
 	github.com/metachris/flashbotsrpc v0.5.0
 	github.com/metachris/go-ethutils v0.5.3
 	github.com/olekukonko/tablewriter v0.0.5

go.sum

@@ -470,6 +470,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w=
+github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=

scripts/bidcollect/bids-combine-and-upload-yesterday.sh

@@ -27,4 +27,4 @@ source .env.prod
 ./scripts/bidcollect/bids-combine-and-upload.sh "/mnt/data/relayscan-bids/$d/"
 
 # update website
-make update-bids-website
+./relayscan service bidcollect --build-website --build-website-upload

services/bidcollect/bid-processor.go

@@ -20,6 +20,7 @@ import (
 
 type BidProcessorOpts struct {
 	Log       *logrus.Entry
+	UID       string
 	OutDir    string
 	OutputTSV bool
 }
@@ -202,7 +203,7 @@ func (c *BidProcessor) getFilename(prefix string, timestamp int64) string {
 	if prefix != "" {
 		prefix += "_"
 	}
-	return fmt.Sprintf("%s%s.%s", prefix, t.Format("2006-01-02_15-04"), c.csvFileEnding)
+	return fmt.Sprintf("%s%s_%s.%s", prefix, t.Format("2006-01-02_15-04"), c.opts.UID, c.csvFileEnding)
 }
 
 func (c *BidProcessor) housekeeping() {

services/bidcollect/bidcollector.go

@@ -8,6 +8,7 @@ import (
 
 type BidCollectorOpts struct {
 	Log *logrus.Entry
+	UID string
 
 	CollectUltrasoundStream bool
 	CollectGetHeader        bool
@@ -49,6 +50,7 @@ func NewBidCollector(opts *BidCollectorOpts) *BidCollector {
 	// output
 	c.processor = NewBidProcessor(&BidProcessorOpts{
 		Log:       opts.Log,
+		UID:       opts.UID,
 		OutDir:    opts.OutDir,
 		OutputTSV: opts.OutputTSV,
 	})