From 6f584de5e07936d94babf99a43a151c47aaecf47 Mon Sep 17 00:00:00 2001 From: Chris Hager Date: Fri, 14 Jun 2024 09:33:31 +0200 Subject: [PATCH] bidcollect: fn with uid --- cmd/service/bidcollect.go | 14 +++++++++++-- docs/2024-06_bidcollect.md | 20 +++++++++++-------- go.mod | 1 + go.sum | 2 ++ .../bids-combine-and-upload-yesterday.sh | 2 +- services/bidcollect/bid-processor.go | 3 ++- services/bidcollect/bidcollector.go | 2 ++ 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go index b444423..3650a6f 100644 --- a/cmd/service/bidcollect.go +++ b/cmd/service/bidcollect.go @@ -9,6 +9,7 @@ import ( "github.com/flashbots/relayscan/services/bidcollect" "github.com/flashbots/relayscan/services/bidcollect/website" "github.com/flashbots/relayscan/vars" + "github.com/lithammer/shortuuid" "github.com/spf13/cobra" ) @@ -19,7 +20,8 @@ var ( useAllRelays bool outDir string - outputTSV bool // by default: CSV, but can be changed to TSV with this setting + outputTSV bool // by default: CSV, but can be changed to TSV with this setting + uid string // used in output filenames, to avoid collissions between multiple collector instances runDevServerOnly bool // used to play with file listing website devServerListenAddr string @@ -42,6 +44,9 @@ func init() { bidCollectCmd.Flags().StringVar(&outDir, "out", "csv", "output directory for CSV/TSV") bidCollectCmd.Flags().BoolVar(&outputTSV, "out-tsv", false, "output as TSV (instead of CSV)") + // utils + bidCollectCmd.Flags().StringVar(&uid, "uid", "", "unique identifier for output files (to avoid collisions)") + // for dev purposes bidCollectCmd.Flags().BoolVar(&runDevServerOnly, "devserver", false, "only run devserver to play with file listing website") bidCollectCmd.Flags().StringVar(&devServerListenAddr, "devserver-addr", "localhost:8095", "listen address for devserver") @@ -68,7 +73,11 @@ var bidCollectCmd = &cobra.Command{ return } - log.Infof("Bidcollect %s starting ...", vars.Version) + if uid == "" { + uid = shortuuid.New()[:6] + } + + log.WithField("uid", uid).Infof("Bidcollect %s starting ...", vars.Version) // Prepare relays relays := []common.RelayEntry{ @@ -86,6 +95,7 @@ var bidCollectCmd = &cobra.Command{ opts := bidcollect.BidCollectorOpts{ Log: log, + UID: uid, Relays: relays, CollectUltrasoundStream: collectUltrasoundStream, CollectGetHeader: collectGetHeader, diff --git a/docs/2024-06_bidcollect.md b/docs/2024-06_bidcollect.md index 15a400d..1873010 100644 --- a/docs/2024-06_bidcollect.md +++ b/docs/2024-06_bidcollect.md @@ -96,19 +96,23 @@ go run . service bidcollect --get-header --beacon-uri http://localhost:3500 --al Useful [clickhouse-local](https://clickhouse.com/docs/en/operations/utilities/clickhouse-local) queries: ```bash -$ clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '2024-06-12_top.csv' GROUP BY source_type ORDER BY source_type;" +# Set the CSV filename, for ease of reuse across queries +$ fn=2024-06-12_top.csv + +# Count different source types in file +$ clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '$fn' GROUP BY source_type ORDER BY source_type;" 0 2929 1 21249 2 1057722 -# Get bids > 1 ETH for specific builders (CSV has 10M rows) -$ time clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '2024-06-05_all.csv' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0xa01a00479f1fa442a8ebadb352be69091d07b0c0a733fae9166dae1b83179e326a968717da175c7363cd5a13e8580e8d', '0xa02a0054ea4ba422c88baccfdb1f43b2c805f01d1475335ea6647f69032da847a41c0e23796c6bed39b0ee11ab9772c6', '0xa03a000b0e3d1dc008f6075a1b1af24e6890bd674c26235ce95ac06e86f2bd3ccf4391df461b9e5d3ca654ef6b9e1ceb') FORMAT TabSeparatedWithNames;" -count(value) p50 p75 p90 max(value) -1842 1789830446982354000 2279820737908906200 4041286254343376400 8216794401676997763 +# Count optimistic mode +$ clickhouse local -q "SELECT optimistic_submission, COUNT(optimistic_submission) FROM '$fn' WHERE optimistic_submission IS NOT NULL GROUP BY optimistic_submission;" + +# Count bids with >1 ETH in value, by builder +$ clickhouse local -q "SELECT builder_pubkey, count(builder_pubkey) as count, quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '$fn' WHERE value > 1000000000000000000 AND builder_pubkey != '' GROUP BY builder_pubkey ORDER BY count DESC FORMAT TabSeparatedWithNames;" -real 0m2.202s -user 0m17.320s -sys 0m0.589s +# Get bids > 1 ETH for specific builders +$ clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '$fn' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0x...', '0x...', '0x...') FORMAT TabSeparatedWithNames;" ``` --- diff --git a/go.mod b/go.mod index b59dee0..c2ab8d5 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.9 + github.com/lithammer/shortuuid v3.0.0+incompatible github.com/metachris/flashbotsrpc v0.5.0 github.com/metachris/go-ethutils v0.5.3 github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 2e8b7de..1d61771 100644 --- a/go.sum +++ b/go.sum @@ -470,6 +470,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/shortuuid v3.0.0+incompatible h1:NcD0xWW/MZYXEHa6ITy6kaXN5nwm/V115vj2YXfhS0w= +github.com/lithammer/shortuuid v3.0.0+incompatible/go.mod h1:FR74pbAuElzOUuenUHTK2Tciko1/vKuIKS9dSkDrA4w= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= diff --git a/scripts/bidcollect/bids-combine-and-upload-yesterday.sh b/scripts/bidcollect/bids-combine-and-upload-yesterday.sh index aa7fae1..3c9a3f8 100755 --- a/scripts/bidcollect/bids-combine-and-upload-yesterday.sh +++ b/scripts/bidcollect/bids-combine-and-upload-yesterday.sh @@ -27,4 +27,4 @@ source .env.prod ./scripts/bidcollect/bids-combine-and-upload.sh "/mnt/data/relayscan-bids/$d/" # update website -make update-bids-website +./relayscan service bidcollect --build-website --build-website-upload diff --git a/services/bidcollect/bid-processor.go b/services/bidcollect/bid-processor.go index 47c2ac7..556bc4f 100644 --- a/services/bidcollect/bid-processor.go +++ b/services/bidcollect/bid-processor.go @@ -20,6 +20,7 @@ import ( type BidProcessorOpts struct { Log *logrus.Entry + UID string OutDir string OutputTSV bool } @@ -202,7 +203,7 @@ func (c *BidProcessor) getFilename(prefix string, timestamp int64) string { if prefix != "" { prefix += "_" } - return fmt.Sprintf("%s%s.%s", prefix, t.Format("2006-01-02_15-04"), c.csvFileEnding) + return fmt.Sprintf("%s%s_%s.%s", prefix, t.Format("2006-01-02_15-04"), c.opts.UID, c.csvFileEnding) } func (c *BidProcessor) housekeeping() { diff --git a/services/bidcollect/bidcollector.go b/services/bidcollect/bidcollector.go index 7c16de3..16c7625 100644 --- a/services/bidcollect/bidcollector.go +++ b/services/bidcollect/bidcollector.go @@ -8,6 +8,7 @@ import ( type BidCollectorOpts struct { Log *logrus.Entry + UID string CollectUltrasoundStream bool CollectGetHeader bool @@ -49,6 +50,7 @@ func NewBidCollector(opts *BidCollectorOpts) *BidCollector { // output c.processor = NewBidProcessor(&BidProcessorOpts{ Log: opts.Log, + UID: opts.UID, OutDir: opts.OutDir, OutputTSV: opts.OutputTSV, })