cleanup

flashbots · Jun 13, 2024 · 3dcf2e6 · 3dcf2e6
1 parent d9ec5db
commit 3dcf2e6
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 18 deletions.
diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go
@@ -56,7 +56,7 @@ var bidCollectCmd = &cobra.Command{
 	Short: "Collect bids",
 	Run: func(cmd *cobra.Command, args []string) {
 		if runDevServerOnly {
-			log.Infof("Bidcollect devserver starting (%s) ...", vars.Version)
+			log.Infof("Bidcollect (%s) devserver starting on %s ...", vars.Version, devServerListenAddr)
 			fileListingDevServer()
 			return
 		}

diff --git a/docs/2024-06_bidcollect.md b/docs/2024-06_bidcollect.md
@@ -1,6 +1,6 @@
 # Bid Collection
 
-Relayscan should collect bids across relays:
+Relayscan collects bids across [relays](../vars/relays.go) with these methods:
 
 1. [getHeader polling](https://ethereum.github.io/builder-specs/#/Builder/getHeader)
 2. [Data API polling](https://flashbots.github.io/relay-specs/#/Data/getReceivedBids)
@@ -9,13 +9,12 @@ Relayscan should collect bids across relays:
 Output:
 
 1. CSV file archive
-2. Websocket/SSE stream (maybe)
 
 See also:
 
 - [Example output](https://gist.github.com/metachris/061c0443afb8b8d07eed477a848fa395)
-- PR: https://github.com/flashbots/relayscan/pull/37
-- TODO: link CSV files
+- [Pull request #37](https://github.com/flashbots/relayscan/pull/37)
+- Live data: https://bidarchive.relayscan.io
 
 ---
 
@@ -29,18 +28,18 @@ Source types:
 Different data sources have different limitations:
 
 - `getHeader` polling:
-  - Some relays only allow a single `getHeader` request per slot, so we time it at t=1s
-  - Header only has limited information with these implications:
+  - The received header only has limited information, with these implications:
     - Optimistic is always `false`
-    - Does not include `builder_pubkey`
-    - Does not include bid timestamp (need to use receive timestamp)
+    - No `builder_pubkey`
+    - No bid timestamp (need to use receive timestamp)
     - getHeader bid timestamps are always when the response from polling at t=1s comes back (but not when the bid was received at a relay)
+  - Some relays only allow a single `getHeader` request per slot, so we time it at `t=1s`
 - Data API polling:
     - Has all the necessary information
     - Due to rate limits, we only poll at specific times
     - Polling at t-4, t-2, t-0.5, t+0.5, t+2 (see also [`services/bidcollect/data-api-poller.go`](services/bidcollect/data-api-poller.go#64-69))
-  - Ultrasound websocket stream
-    - doesn't expose optimistic, thus that field is always `false`
+- Ultrasound websocket stream
+  - doesn't expose optimistic, thus that field is always `false`
 
 ## Other notes
 
@@ -69,10 +68,13 @@ go run . service bidcollect --get-header --beacon-uri http://localhost:3500 --al
 Useful [clickhouse-local](https://clickhouse.com/docs/en/operations/utilities/clickhouse-local) queries:
 
 ```bash
-clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '2024-06-02_top-00.tsv' GROUP BY source_type ORDER BY source_type;"
+$ clickhouse local -q "SELECT source_type, COUNT(source_type) FROM '2024-06-12_top.csv' GROUP BY source_type ORDER BY source_type;"
+0       2929
+1       21249
+2       1057722
 
 # Get bids > 1 ETH for specific builders (CSV has 10M rows)
-time clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '2024-06-05_all.csv' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0xa01a00479f1fa442a8ebadb352be69091d07b0c0a733fae9166dae1b83179e326a968717da175c7363cd5a13e8580e8d', '0xa02a0054ea4ba422c88baccfdb1f43b2c805f01d1475335ea6647f69032da847a41c0e23796c6bed39b0ee11ab9772c6', '0xa03a000b0e3d1dc008f6075a1b1af24e6890bd674c26235ce95ac06e86f2bd3ccf4391df461b9e5d3ca654ef6b9e1ceb') FORMAT TabSeparatedWithNames;"
+$ time clickhouse local -q "SELECT count(value), quantile(0.5)(value) as p50, quantile(0.75)(value) as p75, quantile(0.9)(value) as p90, max(value) FROM '2024-06-05_all.csv' WHERE value > 1000000000000000000 AND builder_pubkey IN ('0xa01a00479f1fa442a8ebadb352be69091d07b0c0a733fae9166dae1b83179e326a968717da175c7363cd5a13e8580e8d', '0xa02a0054ea4ba422c88baccfdb1f43b2c805f01d1475335ea6647f69032da847a41c0e23796c6bed39b0ee11ab9772c6', '0xa03a000b0e3d1dc008f6075a1b1af24e6890bd674c26235ce95ac06e86f2bd3ccf4391df461b9e5d3ca654ef6b9e1ceb') FORMAT TabSeparatedWithNames;"
 count(value)    p50     p75     p90     max(value)
 1842    1789830446982354000     2279820737908906200     4041286254343376400     8216794401676997763
 
@@ -92,5 +94,4 @@ sys     0m0.589s
 
 ## TODO
 
-- Website generation
 - Dockerization
diff --git a/services/bidcollect/website/templates/base.html b/services/bidcollect/website/templates/base.html
@@ -24,21 +24,21 @@
 
     <!-- HTML Meta Tags -->
     <title>{{ $title }}</title>
-    <meta name="description" content="Free Ethereum mempool transactions archive, brought to you by Flashbots ⚡️🤖">
+    <meta name="description" content="MEV-Boost bid archive across relays, brought to you by Flashbots ⚡️🤖">
 
     <!-- Facebook Meta Tags -->
     <meta property="og:url" content="https://relayscan-bidarchive.flashbots.net{{ .Path }}">
     <meta property="og:type" content="website">
     <meta property="og:title" content="{{ $title }}">
-    <meta property="og:description" content="Free MEV-Boost Bid Archive, brought to you by Flashbots ⚡️🤖">
+    <meta property="og:description" content="MEV-Boost bid archive across relays, brought to you by Flashbots ⚡️🤖">
     <meta property="og:image" content="https://relayscan-bidarchive.flashbots.net/static/favicon/android-chrome-512x512.png">
 
     <!-- Twitter Meta Tags -->
     <meta name="twitter:card" content="summary">
     <meta property="twitter:domain" content="relayscan-bidarchive.flashbots.net">
     <meta property="twitter:url" content="https://relayscan-bidarchive.flashbots.net{{ .Path }}">
     <meta name="twitter:title" content="{{ $title }}">
-    <meta name="twitter:description" content="Free MEV-Boost Bid Archive, brought to you by Flashbots ⚡️🤖">
+    <meta name="twitter:description" content="MEV-Boost bid archive across relays, brought to you by Flashbots ⚡️🤖">
     <meta name="twitter:image" content="https://relayscan-bidarchive.flashbots.net/static/favicon/android-chrome-512x512.png">
 
     <style>
@@ -113,7 +113,8 @@
         </a>
         <h1>{{ $title }}</h1>
         <p>
-            <a href="https://github.com/flashbots/relayscan">https://github.com/flashbots/relayscan</a>
+            <!-- <a href="https://github.com/flashbots/relayscan">https://github.com/flashbots/relayscan</a> &middot; <a href="https://github.com/flashbots/relayscan/blob/bidstream/docs/2024-06_bidcollect.md">Bid Archive Docs</a> -->
+            <a href="https://github.com/flashbots/relayscan/blob/bidstream/docs/2024-06_bidcollect.md">docs</a> &middot; <a href="https://github.com/flashbots/relayscan">https://github.com/flashbots/relayscan</a>
         </p>
         <p><i>Illuminate, Democratize, Distribute</i></p>
     </div>