data_collection.rs stability improvements
#656
Conversation
Codecov Report
@@ Coverage Diff @@
## main #656 +/- ##
==========================================
- Coverage 60.76% 60.67% -0.09%
==========================================
Files 26 26
Lines 5533 5551 +18
==========================================
+ Hits 3362 3368 +6
- Misses 2171 2183 +12
📣 We’re building smart automated test selection to slash your CI/CD build times. Learn more |
There was a problem hiding this comment.
Let me know if I am getting this right on whats going on here now:
- We made a new dedicated broadcast channel
- We have that channel run on a separate normal thread
- We push each event through this channel and store it in the same place so its output to the same file
| while let Some(res) = set.join_next().await { | ||
| info!("task completed: {:?}", res); | ||
| let receiver = self.receiver.unwrap(); | ||
| let dir = self.directory.unwrap_or("./out".into()); |
There was a problem hiding this comment.
Does this overwrite if an existing directory is there? Might conflict with default foundry output which is out, if it does an it overwrites the foundry out that could make issues
There was a problem hiding this comment.
Good catch. @kinrezC let's change this do something like ./data_output
| return Ok(()); | ||
| } else { | ||
| if logs.is_none() { | ||
| panic!("This shouldn't happen, but we should probably make this an error. Somehow we got told to broadcast `None` logs!"); |
There was a problem hiding this comment.
lol is this possible to hit?
There was a problem hiding this comment.
I really don't think so. I think if no logs, you actually broadcast vec![]. That could actually be optimized out...
| tokio::fs::remove_dir_all("./test_output1").await.unwrap(); | ||
|
|
||
| env.stop(); | ||
| tokio::fs::remove_dir_all("./out").await.unwrap(); | ||
| } | ||
|
|
||
| #[traced_test] | ||
| #[tokio::test(flavor = "multi_thread")] | ||
| async fn data_capture_output_validation() { |
| @@ -0,0 +1 @@ | |||
| {"arbx":{"ApprovalFilter":[{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"}]},"arby":{"ApprovalFilter":[{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"},{"owner":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","spender":"0x2efdc9eecfee3a776209fcb8e9a83a6b221d74f5","amount":"0x1"}]},"lex":{"PriceChangeFilter":[{"price":"0xde0b6b3a7640000"},{"price":"0xde0b6b3a7640000"},{"price":"0xde0b6b3a7640000"},{"price":"0xde0b6b3a7640000"},{"price":"0xde0b6b3a7640000"}]}} | |||
There was a problem hiding this comment.
okay with the labels is nice
There was a problem hiding this comment.
And unraveling it is super clean!
| // Grab the connection from the client and add a new event sender so that we | ||
| // have a distinct channel to now receive events over | ||
| let event_transmuted: EventTransmuted<Arc<RevmMiddleware>, RevmMiddleware, D> = | ||
| unsafe { transmute(event) }; |
There was a problem hiding this comment.
idk what this does or why its here, maybe a comment?
There was a problem hiding this comment.
Same, is the kinda same unsafe stuff we did with resolving pending transactions?
There was a problem hiding this comment.
lets you access private fields
There was a problem hiding this comment.
@Alexangelj if you're talking about the transmute, it is essentially what Matt said.
So you can take an Event<Arc<RevmMiddleware>, RevmMiddleware, D> and cast it into the EventTransmuted<Arc<RevmMiddleware, RevmMiddleware, D> which then gives us direct access to the fields of the struct.
data_collection.rsto avoid any race conditions. This PR makes the whole event logger process sync and adds a signal from the environment to indicate when the process should shut down.cargo test data_outputfails #590