Merge pull request #381 from XxChang/main

add cmake example

.github/workflows/ci.yml

@@ -131,6 +131,10 @@ jobs:
       - name: "C++ Dataflow example"
         timeout-minutes: 15
         run: cargo run --example cxx-dataflow
+      - name: "Cmake example"
+        if: runner.os == 'Linux'
+        timeout-minutes: 30
+        run: cargo run --example cmake-dataflow
 
       # python examples
       - uses: actions/setup-python@v2

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 members = [
-    "apis/c/*",
-    "apis/c++/*",
+    "apis/c/node",
+    "apis/c/operator",
+    "apis/c++/node",
+    "apis/c++/operator",
     "apis/python/node",
     "apis/python/operator",
     "apis/rust/*",
@@ -133,3 +135,7 @@ path = "examples/benchmark/run.rs"
 [[example]]
 name = "multiple-daemons"
 path = "examples/multiple-daemons/run.rs"
+
+[[example]]
+name = "cmake-dataflow"
+path = "examples/cmake-dataflow/run.rs"

examples/cmake-dataflow/.gitignore

@@ -0,0 +1,6 @@
+*.o
+
+# cmake artifact
+build
+lib
+bin

examples/cmake-dataflow/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required(VERSION 3.21)
+project(cmake-dataflow LANGUAGES C CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_FLAGS "-fPIC")
+
+include(DoraTargets.cmake)
+
+link_directories(${dora_link_dirs})
+
+add_executable(node_c_api node-c-api/main.cc)
+add_dependencies(node_c_api Dora_c)
+target_include_directories(node_c_api PRIVATE ${dora_c_include_dir})
+target_link_libraries(node_c_api dora_node_api_c)
+
+add_executable(node_rust_api node-rust-api/main.cc ${node_bridge})
+add_dependencies(node_rust_api Dora_cxx)
+target_include_directories(node_rust_api PRIVATE ${dora_cxx_include_dir})
+target_link_libraries(node_rust_api dora_node_api_cxx)
+
+add_library(operator_c_api SHARED operator-c-api/operator.cc)
+add_dependencies(operator_c_api Dora_c)
+target_include_directories(operator_c_api PRIVATE ${dora_c_include_dir})
+target_link_libraries(operator_c_api dora_operator_api_c)
+
+add_library(operator_rust_api SHARED operator-rust-api/operator.cc ${operator_bridge})
+add_dependencies(operator_rust_api Dora_cxx)
+target_include_directories(operator_rust_api PRIVATE ${dora_cxx_include_dir} ${dora_c_include_dir} ${CMAKE_CURRENT_SOURCE_DIR}/operator-rust-api)
+target_link_libraries(operator_rust_api dora_operator_api_cxx)
+
+install(TARGETS node_c_api node_rust_api DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+install(TARGETS operator_c_api operator_rust_api DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/lib)

examples/cmake-dataflow/DoraTargets.cmake

@@ -0,0 +1,119 @@
+set(DORA_ROOT_DIR "" CACHE FILEPATH "Path to the root of dora")
+
+set(dora_c_include_dir "${CMAKE_CURRENT_BINARY_DIR}/include/c")
+
+set(dora_cxx_include_dir "${CMAKE_CURRENT_BINARY_DIR}/include/cxx")
+set(node_bridge "${CMAKE_CURRENT_BINARY_DIR}/node_bridge.cc")
+set(operator_bridge "${CMAKE_CURRENT_BINARY_DIR}/operator_bridge.cc")
+
+if(DORA_ROOT_DIR)
+    include(FetchContent)
+    FetchContent_Declare(
+        Corrosion
+        GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
+        GIT_TAG v0.4.3
+    )
+    FetchContent_MakeAvailable(Corrosion)
+    list(PREPEND CMAKE_MODULE_PATH ${Corrosion_SOURCE_DIR}/cmake)
+    find_package(Rust 1.70 REQUIRED MODULE)
+    corrosion_import_crate(MANIFEST_PATH "${DORA_ROOT_DIR}/Cargo.toml"
+        CRATES
+        dora-node-api-c
+        dora-operator-api-c
+        CRATE_TYPES
+        staticlib staticlib
+    )
+    add_custom_command(OUTPUT ${dora_c_include_dir}
+        WORKING_DIRECTORY ${DORA_ROOT_DIR}/apis/c
+        COMMAND
+            mkdir ${CMAKE_CURRENT_BINARY_DIR}/include/c -p
+            &&
+            cp node ${CMAKE_CURRENT_BINARY_DIR}/include/c -r
+            &&
+            cp operator ${CMAKE_CURRENT_BINARY_DIR}/include/c -r
+        DEPENDS dora-node-api-c dora-operator-api-c
+    )
+
+    corrosion_import_crate(MANIFEST_PATH "${DORA_ROOT_DIR}/Cargo.toml"
+        CRATES
+        dora-node-api-cxx
+        dora-operator-api-cxx
+        CRATE_TYPES
+        staticlib staticlib
+    )
+    add_custom_command(OUTPUT ${node_bridge} ${dora_cxx_include_dir} ${operator_bridge}
+        WORKING_DIRECTORY ${DORA_ROOT_DIR}
+        DEPENDS dora-node-api-cxx dora-operator-api-cxx
+        COMMAND
+            mkdir ${dora_cxx_include_dir} -p
+            &&
+            cp target/cxxbridge/dora-node-api-cxx/src/lib.rs.cc ${node_bridge}
+            &&
+            cp target/cxxbridge/dora-node-api-cxx/src/lib.rs.h ${dora_cxx_include_dir}/dora-node-api.h
+            &&
+            cp target/cxxbridge/dora-operator-api-cxx/src/lib.rs.cc ${operator_bridge}
+            &&
+            cp target/cxxbridge/dora-operator-api-cxx/src/lib.rs.h ${dora_cxx_include_dir}/dora-operator-api.h
+    )
+
+    add_custom_target(Dora_c DEPENDS dora-node-api-c dora-operator-api-c ${dora_c_include_dir})
+    add_custom_target(Dora_cxx DEPENDS dora-node-api-cxx  dora-operator-api-cxx ${node_bridge} ${operator_bridge} ${dora_cxx_include_dir})
+    set(dora_link_dirs ${CMAKE_CURRENT_BINARY_DIR})
+else()
+    include(ExternalProject)
+    ExternalProject_Add(Dora
+        PREFIX ${CMAKE_CURRENT_BINARY_DIR}/dora
+        GIT_REPOSITORY https://github.com/dora-rs/dora.git
+        GIT_TAG main
+        BUILD_IN_SOURCE True
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND
+            cargo build
+            --package dora-node-api-c
+            --target-dir ${CMAKE_CURRENT_BINARY_DIR}
+            &&
+            cargo build
+            --package dora-operator-api-c
+            --target-dir ${CMAKE_CURRENT_BINARY_DIR}
+            &&
+            cargo build
+            --package dora-node-api-cxx
+            --target-dir ${CMAKE_CURRENT_BINARY_DIR}
+            &&
+            cargo build
+            --package dora-operator-api-cxx
+            --target-dir ${CMAKE_CURRENT_BINARY_DIR}
+        INSTALL_COMMAND ""
+    )
+    add_custom_command(OUTPUT ${dora_c_include_dir}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dora/src/Dora/apis/c
+        COMMAND
+            mkdir ${CMAKE_CURRENT_BINARY_DIR}/include/c -p
+            &&
+            cp node ${CMAKE_CURRENT_BINARY_DIR}/include/c -r
+            &&
+            cp operator ${CMAKE_CURRENT_BINARY_DIR}/include/c -r
+        DEPENDS Dora
+    )
+    add_custom_command(OUTPUT ${node_bridge} ${dora_cxx_include_dir} ${operator_bridge}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDS Dora
+        COMMAND
+            mkdir ${dora_cxx_include_dir} -p
+            &&
+            cp cxxbridge/dora-node-api-cxx/src/lib.rs.cc ${node_bridge}
+            &&
+            cp cxxbridge/dora-node-api-cxx/src/lib.rs.h ${dora_cxx_include_dir}/dora-node-api.h
+            &&
+            cp cxxbridge/dora-operator-api-cxx/src/lib.rs.cc ${operator_bridge}
+            &&
+            cp cxxbridge/dora-operator-api-cxx/src/lib.rs.h ${dora_cxx_include_dir}/dora-operator-api.h
+    )
+
+    set(dora_link_dirs ${CMAKE_CURRENT_BINARY_DIR}/debug)
+
+    add_custom_target(Dora_c DEPENDS ${dora_c_include_dir})
+    add_custom_target(Dora_cxx DEPENDS ${node_bridge} ${operator_bridge} ${dora_cxx_include_dir})
+endif()
+
+

examples/cmake-dataflow/README.md

@@ -0,0 +1,22 @@
+# Dora CMake Dataflow Example
+
+This example shows how to create dora operators and custom nodes in CMake build system.
+
+See also [c++-example](https://github.com/dora-rs/dora/blob/main/examples/c%2B%2B-dataflow/README.md) for the implementation details of operator and node.
+
+## Compile and Run
+
+To try it out, you can use the [`run.rs`](./run.rs) binary. It performs all required build steps and then starts the dataflow. Use the following command to run it: `cargo run --example cmake-dataflow`.
+
+## Out-of-tree complie
+
+This example also can be ran in a separate root directory.
+```
+cd <path-to-cmake-dataflow>
+mkdir build
+cd build && cmake ..
+make install
+cd ..
+dora up
+dora start dataflow.yml
+```

examples/cmake-dataflow/dataflow.yml

@@ -0,0 +1,33 @@
+nodes:
+  - id: cxx-node-rust-api
+    custom:
+      source: bin/node_rust_api
+      inputs:
+        tick: dora/timer/millis/300
+      outputs:
+        - counter
+  - id: cxx-node-c-api
+    custom:
+      source: bin/node_c_api
+      inputs:
+        tick: dora/timer/millis/300
+      outputs:
+        - counter
+
+  - id: runtime-node-1
+    operators:
+      - id: operator-rust-api
+        shared-library: lib/operator_rust_api
+        inputs:
+          counter_1: cxx-node-c-api/counter
+          counter_2: cxx-node-rust-api/counter
+        outputs:
+          - status
+  - id: runtime-node-2
+    operators:
+      - id: operator-c-api
+        shared-library: lib/operator_c_api
+        inputs:
+          op_status: runtime-node-1/operator-rust-api/status
+        outputs:
+          - half-status

examples/cmake-dataflow/node-c-api/main.cc

@@ -0,0 +1,85 @@
+extern "C"
+{
+#include "node/node_api.h"
+}
+
+#include <iostream>
+#include <vector>
+
+int run(void *dora_context)
+{
+    unsigned char counter = 0;
+
+    for (int i = 0; i < 20; i++)
+    {
+        void *event = dora_next_event(dora_context);
+        if (event == NULL)
+        {
+            printf("[c node] ERROR: unexpected end of event\n");
+            return -1;
+        }
+
+        enum DoraEventType ty = read_dora_event_type(event);
+
+        if (ty == DoraEventType_Input)
+        {
+            counter += 1;
+
+            char *id_ptr;
+            size_t id_len;
+            read_dora_input_id(event, &id_ptr, &id_len);
+            std::string id(id_ptr, id_len);
+
+            char *data_ptr;
+            size_t data_len;
+            read_dora_input_data(event, &data_ptr, &data_len);
+            std::vector<unsigned char> data;
+            for (size_t i = 0; i < data_len; i++)
+            {
+                data.push_back(*(data_ptr + i));
+            }
+
+            std::cout
+                << "Received input "
+                << " (counter: " << (unsigned int)counter << ") data: [";
+            for (unsigned char &v : data)
+            {
+                std::cout << (unsigned int)v << ", ";
+            }
+            std::cout << "]" << std::endl;
+
+            std::vector<unsigned char> out_vec{counter};
+            std::string out_id = "counter";
+            int result = dora_send_output(dora_context, &out_id[0], out_id.length(), (char *)&counter, 1);
+            if (result != 0)
+            {
+                std::cerr << "failed to send output" << std::endl;
+                return 1;
+            }
+        }
+        else if (ty == DoraEventType_Stop)
+        {
+            printf("[c node] received stop event\n");
+        }
+        else
+        {
+            printf("[c node] received unexpected event: %d\n", ty);
+        }
+
+        free_dora_event(event);
+    }
+    return 0;
+}
+
+int main()
+{
+    std::cout << "HELLO FROM C++ (using C API)" << std::endl;
+
+    auto dora_context = init_dora_context_from_env();
+    auto ret = run(dora_context);
+    free_dora_context(dora_context);
+
+    std::cout << "GOODBYE FROM C++ node (using C API)" << std::endl;
+
+    return ret;
+}

examples/cmake-dataflow/node-rust-api/main.cc

@@ -0,0 +1,50 @@
+#include "dora-node-api.h"
+
+#include <iostream>
+#include <vector>
+
+int main()
+{
+    std::cout << "HELLO FROM C++" << std::endl;
+    unsigned char counter = 0;
+
+    auto dora_node = init_dora_node();
+
+    for (int i = 0; i < 20; i++)
+    {
+
+        auto event = next_event(dora_node.events);
+        auto ty = event_type(event);
+
+        if (ty == DoraEventType::AllInputsClosed)
+        {
+            break;
+        }
+        else if (ty == DoraEventType::Input)
+        {
+            auto input = event_as_input(std::move(event));
+
+            counter += 1;
+
+            std::cout << "Received input " << std::string(input.id) << " (counter: " << (unsigned int)counter << ")" << std::endl;
+
+            std::vector<unsigned char> out_vec{counter};
+            rust::Slice<const uint8_t> out_slice{out_vec.data(), out_vec.size()};
+            auto result = send_output(dora_node.send_output, "counter", out_slice);
+            auto error = std::string(result.error);
+            if (!error.empty())
+            {
+                std::cerr << "Error: " << error << std::endl;
+                return -1;
+            }
+        }
+        else
+        {
+            std::cerr << "Unknown event type " << static_cast<int>(ty) << std::endl;
+        }
+    }
+
+    std::cout << "GOODBYE FROM C++ node (using Rust API)" << std::endl;
+
+    return 0;
+}