Many protocol adjustments

.github/workflows/ci.yaml

@@ -38,6 +38,7 @@ jobs:
       markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }}
       vscode_changed: ${{ steps.changed-vscode.outputs.any_changed }}
       ci_changed: ${{ steps.changed-ci.outputs.any_changed }}
+      protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -80,6 +81,24 @@ jobs:
         with:
           files: |
             README.md
+      - name: get protocol changes
+        id: changed-protocol
+        uses: tj-actions/changed-files@v36
+        with:
+          files: |
+            mirrord/protocol/**
+      - name: get protocol changes
+        id: changed-protocol-toml
+        uses: tj-actions/changed-files@v36
+        with:
+          files: |
+            mirrord/protocol/Cargo.toml
+      - name: verify protocol bump
+        run: |
+          if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then
+            echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml."
+            exit 1
+          fi
       - name: output test
         run: |
           echo ${{ steps.changed-rs.outputs.any_changed }};
@@ -92,6 +111,8 @@ jobs:
           echo ${{ steps.changed-markdown.outputs.all_changed_files }};    
           echo ${{ steps.changed-ci.outputs.any_changed }};
           echo ${{ steps.changed-ci.outputs.all_changed_files }};
+          echo ${{ steps.changed-protocol.outputs.any_changed }};
+          echo ${{ steps.changed-protocol-toml.outputs.any_changed }};
   lint:
     runs-on: ubuntu-latest
     needs: changed_files
@@ -579,6 +600,7 @@ jobs:
         with:
           config: "markdownlint-config.json"
           args: "README.md"
+
   # We need some "accummulation" job here because bors fails (timeouts) to
   # listen on matrix builds.
   # Hence, we have some kind of dummy here that bors can listen on

changelog.d/1355.internal.md

@@ -0,0 +1,3 @@
+change `mirrord-protocol` to have its own versioning. add `mirrord-macros` and `protocol_break` attribute to mark places we want to break on major updates.
+Add CI to verify that if protocol is changed, `Cargo.toml` is changed as well (to force bumps)
+Fix some of the structs being `OS` controlled, potentially breaking the protocol between different OS's. (`GetDEnts64(RemoteResult<GetDEnts64Response>),`)

mirrord/layer/src/file.rs

@@ -225,6 +225,13 @@ impl FileHandler {
                 trace!("DaemonMessage::GetDEnts64Response {getdents64:#?}!");
                 pop_send(&mut self.getdents64_queue, getdents64)
             }
+            #[cfg(not(target_os = "linux"))]
+            GetDEnts64(_) => {
+                error!(
+                    "Received GetDEnts64Response on non-linux platform! Please report this to us!"
+                );
+                Ok(())
+            }
         }
     }
 

mirrord/macros/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "mirrord-macros"
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+documentation.workspace = true
+readme.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+publish.workspace = true
+edition.workspace = true
+
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+proc-macro2-diagnostics = "0.10"
+syn = { version = "1", features = ["full", "extra-traits"] }
+semver.workspace = true

mirrord/macros/src/lib.rs

@@ -0,0 +1,37 @@
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use proc_macro2_diagnostics::SpanDiagnosticExt;
+use semver::Version;
+use syn::{parse_macro_input, spanned::Spanned};
+
+/// Use [`protocol_break`] to mark code that should be revised when the major version is bumped.
+/// For example:
+/// #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)]
+/// pub enum StealType {
+///     All(Port),
+///     FilteredHttp(Port, Filter),
+///     #[protocol_break(2)] // We want that when we bump major into 2, we can remove this variant
+/// or merge it with the above.     FilteredHttpPath(Port, Filter),
+/// }
+#[proc_macro_attribute]
+pub fn protocol_break(attr: TokenStream, input: TokenStream) -> TokenStream {
+    // Get the version to break on
+    let break_on_str = parse_macro_input!(attr as syn::LitInt);
+    let break_on_value = break_on_str.base10_parse::<u64>().unwrap();
+    let input: TokenStream2 = input.into();
+    if Version::parse(&std::env::var("CARGO_PKG_VERSION").unwrap())
+        .unwrap()
+        .major
+        >= break_on_value
+    {
+        input
+            .span()
+            .error("This code should be revised when the major version is bumped.")
+            .emit_as_expr_tokens()
+            .into()
+    } else {
+        input.into()
+    }
+}

mirrord/protocol/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mirrord-protocol"
-version.workspace = true
+version = "1.0.0"
 authors.workspace = true
 description.workspace = true
 documentation.workspace = true
@@ -27,6 +27,9 @@ http-body-util = { workspace = true }
 fancy-regex = { workspace = true }
 libc.workspace = true
 socket2.workspace = true
+semver.workspace = true
+
+mirrord-macros = { path = "../macros" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 nix.workspace = true

mirrord/protocol/src/codec.rs

@@ -6,17 +6,17 @@ use std::{
 use actix_codec::{Decoder, Encoder};
 use bincode::{error::DecodeError, Decode, Encode};
 use bytes::{Buf, BufMut, BytesMut};
+use mirrord_macros::protocol_break;
 
-#[cfg(target_os = "linux")]
-use crate::file::{GetDEnts64Request, GetDEnts64Response};
 use crate::{
     dns::{GetAddrInfoRequest, GetAddrInfoResponse},
     file::{
         AccessFileRequest, AccessFileResponse, CloseDirRequest, CloseFileRequest, FdOpenDirRequest,
-        OpenDirResponse, OpenFileRequest, OpenFileResponse, OpenRelativeFileRequest,
-        ReadDirRequest, ReadDirResponse, ReadFileRequest, ReadFileResponse, ReadLimitedFileRequest,
-        SeekFileRequest, SeekFileResponse, WriteFileRequest, WriteFileResponse,
-        WriteLimitedFileRequest, XstatFsRequest, XstatFsResponse, XstatRequest, XstatResponse,
+        GetDEnts64Request, GetDEnts64Response, OpenDirResponse, OpenFileRequest, OpenFileResponse,
+        OpenRelativeFileRequest, ReadDirRequest, ReadDirResponse, ReadFileRequest,
+        ReadFileResponse, ReadLimitedFileRequest, SeekFileRequest, SeekFileResponse,
+        WriteFileRequest, WriteFileResponse, WriteLimitedFileRequest, XstatFsRequest,
+        XstatFsResponse, XstatRequest, XstatResponse,
     },
     outgoing::{
         tcp::{DaemonTcpOutgoing, LayerTcpOutgoing},
@@ -77,7 +77,6 @@ pub enum FileRequest {
     FdOpenDir(FdOpenDirRequest),
     ReadDir(ReadDirRequest),
     CloseDir(CloseDirRequest),
-    #[cfg(target_os = "linux")]
     GetDEnts64(GetDEnts64Request),
 }
 
@@ -113,11 +112,12 @@ pub enum FileResponse {
     XstatFs(RemoteResult<XstatFsResponse>),
     ReadDir(RemoteResult<ReadDirResponse>),
     OpenDir(RemoteResult<OpenDirResponse>),
-    #[cfg(target_os = "linux")]
     GetDEnts64(RemoteResult<GetDEnts64Response>),
 }
+
 /// `-agent` --> `-layer` messages.
 #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)]
+#[protocol_break(2)]
 pub enum DaemonMessage {
     Close(String),
     Tcp(DaemonTcp),

mirrord/protocol/src/file.rs

@@ -383,14 +383,12 @@ pub struct CloseDirRequest {
     pub remote_fd: u64,
 }
 
-#[cfg(target_os = "linux")]
 #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)]
 pub struct GetDEnts64Request {
     pub remote_fd: u64,
     pub buffer_size: u64,
 }
 
-#[cfg(target_os = "linux")]
 #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)]
 pub struct GetDEnts64Response {
     pub fd: u64,