Merge pull request #862 from andrewdavidmackenzie/flow-861

flow-861 Fixes #861 Fixes #844

.travis.yml

@@ -34,46 +34,40 @@ addons:
     packages:
       - zmq
 
-# Docs are only built and deployed in "cron" builds - so only prepare that when needed
+# Doc building tools are needed on linux with stable rust
 before_install:
   - make no-book-config
   - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "stable" && "$TRAVIS_EVENT_TYPE" == "cron" ]]; then
+    if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "stable" ]]; then
       make book-config
     fi
 
-# Docs are only built and deployed in "cron" builds on linux with rust stable
+# Docs are only built on linux with stable rust
 script:
   - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]; then
-        # Build the docs nightly
-        travis_wait 50 make docs
-      else
-        travis_wait 50 make clippy build test
-      fi
+    if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "stable" ]]; then
+      travis_wait 50 make clippy build test docs
     else
-      if [[ "$TRAVIS_EVENT_TYPE" != "cron" ]]; then
-        travis_wait 50 make clippy test
-      fi
+      travis_wait 50 make clippy test
     fi
 
-# Docs are only built and deployed in "cron" builds on linux with rust stable
+# Docs are only on linux builds of master branch with stable rust
 deploy:
   provider: pages
   skip_cleanup: true
   github-token: $GITHUB_PAGES_TOKEN
   local-dir: target/html
   keep_history: true
-  allow_empty_commit: true # Avoid deploy fail on empty commit if nothing has changed in CRON deploy
+  allow_empty_commit: true # Avoid deploy fail on empty commit if nothing has changed (CRON builds)
   on:
     branch: master
-    condition: $TRAVIS_OS_NAME == "linux" && $TRAVIS_RUST_VERSION == "stable" && $TRAVIS_EVENT_TYPE == "cron"
+    condition: $TRAVIS_OS_NAME == "linux" && $TRAVIS_RUST_VERSION == "stable"
 
 notifications:
   email:
   - [email protected]
 
+# avoid measuring coverage on CRON builds, only measure on non-CRON linux builds with stable rust
 after_success: |
   if [[ "$TRAVIS_OS_NAME" == "linux" && "$TRAVIS_RUST_VERSION" == "stable" && "$TRAVIS_EVENT_TYPE" != "cron" ]]; then
     FLOW_LIB_PATH=$TRAVIS_BUILD_DIR,$TRAVIS_BUILD_DIR/flowr/src/lib make coverage

Makefile

@@ -285,7 +285,10 @@ copy:
 clean:
 	@find . -name \*.dot.svg -type f -exec rm -rf {} + ; true
 	@find . -name \*.dot -type f -exec rm -rf {} + ; true
+	@find . -name manifest.json -type f -exec rm -rf {} + ; true
+	@find . -name test.output -type f -exec rm -rf {} + ; true
+	@find . -name test.err -type f -exec rm -rf {} + ; true
+	@find . -name \*.wasm -type f -exec rm -rf {} + ; true
 	@rm -rf target/html
 	@find . -name \*.dump -type f -exec rm -rf {} + ; true
-	@find . -name \*.dot -type f -exec rm -rf {} + ; true
 	@cargo clean

flowc/src/lib/compiler/compile_wasm.rs

@@ -155,6 +155,7 @@ fn run_cargo_build(manifest_path: &Path, target_dir: &Path) -> Result<String> {
     let mut command_args = vec![
         "build",
         "--quiet",
+        // "--verbose",
         "--lib",
         "--target=wasm32-unknown-unknown",
     ];
@@ -169,7 +170,7 @@ fn run_cargo_build(manifest_path: &Path, target_dir: &Path) -> Result<String> {
     );
 
     let output = Command::new(&command)
-        .args(command_args)
+        .args(&command_args)
         .stdin(Stdio::inherit())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
@@ -179,11 +180,20 @@ fn run_cargo_build(manifest_path: &Path, target_dir: &Path) -> Result<String> {
     match output.status.code() {
         Some(0) => Ok("Cargo Build of supplied function to wasm succeeded".to_string()),
         Some(code) => {
+            error!(
+                "Process STDOUT:\n{}",
+                String::from_utf8_lossy(&output.stdout)
+            );
             error!(
                 "Process STDERR:\n{}",
                 String::from_utf8_lossy(&output.stderr)
             );
-            bail!("Exited with status code: {}", code)
+            bail!(
+                "cargo exited with status code: {}\nCommand Line: {} {:?}",
+                code,
+                command,
+                command_args
+            )
         }
         None => Ok("No return code - ignoring".to_string()),
     }

flowc/src/lib/dumper/dump_dot.rs

@@ -4,12 +4,9 @@ use std::io::prelude::*;
 use std::path::Path;
 
 use serde_json::Value;
-use url::Url;
 
 use flowcore::input::InputInitializer::{Always, Once};
-use provider::lib_provider::LibProvider;
 
-use crate::deserializers::deserializer_helper::get_file_extension;
 use crate::errors::*;
 use crate::generator::generate::GenerationTables;
 use crate::model::connection::Connection;
@@ -37,11 +34,19 @@ fn absolute_to_relative(absolute: &str, current_dir: &Path) -> Result<String> {
     Ok(absolute.replace(&format!("file://{}/", root_path.display()), &path_to_root))
 }
 
+fn remove_file_extension(file_path: &str) -> String {
+    let splits: Vec<&str> = file_path.split('.').collect();
+    if splits.len() > 1 {
+        splits[0..splits.len() - 1].join(".")
+    } else {
+        file_path.to_owned()
+    }
+}
+
 pub fn write_flow_to_dot(
     flow: &Flow,
     dot_file: &mut dyn Write,
     output_dir: &Path,
-    provider: &dyn LibProvider,
 ) -> std::io::Result<()> {
     dot_file.write_all(digraph_wrapper_start(flow).as_bytes())?;
 
@@ -64,29 +69,9 @@ pub fn write_flow_to_dot(
         })?;
         match process {
             FlowProcess(ref flow) => {
-                let (flow_source, _) = provider
-                    .resolve_url(
-                        &Url::parse(&process_ref.source).map_err(|_| {
-                            std::io::Error::new(
-                                std::io::ErrorKind::Other,
-                                "Could not parse Url from flow_source",
-                            )
-                        })?,
-                        "",
-                        &[""],
-                    )
-                    .map_err(|_| {
-                        std::io::Error::new(
-                            std::io::ErrorKind::Other,
-                            "Could not resolve Url of flow_source",
-                        )
-                    })?;
+                // TODO convert lib reference to a file path or url reference to the actual resource
 
-                // remove file extension when forming URL as is of form {file_stem}.dot.svg
-                let mut flow_source_str = flow_source.to_string();
-                if let Some(extension) = get_file_extension(&flow_source) {
-                    flow_source_str.truncate(flow_source_str.len() - (extension.len() + 1))
-                }
+                let flow_source_str = remove_file_extension(&process_ref.source);
 
                 let relative_path =
                     absolute_to_relative(&flow_source_str, output_dir).map_err(|_| {
@@ -444,3 +429,31 @@ pub fn process_refs_to_dot(
 
     Ok(output)
 }
+
+#[cfg(test)]
+mod test {
+    use super::remove_file_extension;
+
+    #[test]
+    fn strip_extension() {
+        assert_eq!("file", remove_file_extension("file.toml"));
+    }
+
+    #[test]
+    fn strip_last_extension_only() {
+        assert_eq!("file.my.file", remove_file_extension("file.my.file.toml"));
+    }
+
+    #[test]
+    fn strip_extension_in_path() {
+        assert_eq!(
+            "/root/home/file",
+            remove_file_extension("/root/home/file.toml")
+        );
+    }
+
+    #[test]
+    fn strip_no_extension() {
+        assert_eq!("file", remove_file_extension("file"));
+    }
+}

flowc/src/lib/dumper/dump_flow.rs

@@ -93,7 +93,7 @@ fn _dump_flow(
 
     writer = dump_tables::create_output_file(&output_dir, filename, "dot")?;
     info!("\tGenerating {}.dot, Use \"dotty\" to view it", filename);
-    dump_dot::write_flow_to_dot(flow, &mut writer, output_dir, provider)?;
+    dump_dot::write_flow_to_dot(flow, &mut writer, output_dir)?;
 
     // Dump sub-flows
     for subprocess in &flow.subprocesses {

flowcore/Cargo.toml

@@ -17,16 +17,21 @@ exclude = [".gitignore", "Cargo.lock"]
 name = "flowcore"
 path = "src/lib.rs"
 
-[features]
-default = []  # Don't include any features by default
-debugger = [] # feature to add the debugger
-
 [dependencies]
-provider = {path = "../provider", version = "0.34.4" }
+serde_json = "1.0" # Needed for Implementation trait and must compile for wasm32-unknown-unknown
 
-url = { version = "2.2", features = ["serde"] }
-serde = "~1.0.27"
-serde_derive = "~1.0.27"
-serde_json = "1.0"
-error-chain = "0.12.2"
-log = "0.4.6"
+# All other dependencies below are optional and can be turned off using "default-features false" removing any
+# dependency that won't compile for wasm32
+serde = { version = "~1.0.27", optional = true }
+serde_derive = { version = "~1.0.27", optional = true }
+provider = {path = "../provider", version = "0.34.4", optional = true}
+url = { version = "2.2", features = ["serde"], optional = true }
+error-chain = {version = "0.12.2", optional = true}
+log = {version = "0.4.6", optional = true}
+
+[features]
+default = ["code"]  # by default include code not just the trait definition
+debugger = ["code"] # feature to add the debugger, depends on code feature
+# Enable the "code" feature to compile the crate with dependencies for code not just trait
+# It brings in the following optional dependencies, some of which do not compile for wasm32
+code = ["serde", "serde_derive", "provider", "url", "error-chain", "log"]

flowcore/src/lib.rs

@@ -1,29 +1,38 @@
 #![warn(clippy::unwrap_used)]
 
-//! `flowcore` create defined some core structs and traits used by other flow libraries
+//! `flowcore` defines some core structs and traits used by other flow libraries and implementations
 
+#[cfg(feature = "code")]
 #[macro_use]
+#[cfg(feature = "code")]
 extern crate error_chain;
 
 use std::panic::{RefUnwindSafe, UnwindSafe};
 
 use serde_json::Value;
 
+#[cfg(feature = "code")]
 /// `function` defines functions that form part of a flow
 pub mod function;
+#[cfg(feature = "code")]
 /// `input` defines the struct for inputs to functions in a flow
 pub mod input;
+#[cfg(feature = "code")]
 /// `lib_manifest` defines the structs for specifying a Library's manifest and methods to load it
 pub mod lib_manifest;
+#[cfg(feature = "code")]
 /// `manifest` is the struct that specifies the manifest of functions in a flow
 pub mod manifest;
+#[cfg(feature = "code")]
 /// `output_connection` defines a struct for a function's output connection
 pub mod output_connection;
+#[cfg(feature = "code")]
 /// Utility functions related to Urls
 pub mod url_helper;
 
 /// We'll put our errors in an `errors` module, and other modules in this crate will `use errors::*;`
 /// to get access to everything `error_chain!` creates.
+#[cfg(feature = "code")]
 #[doc(hidden)]
 pub mod errors {
     // Create the Error, ErrorKind, ResultExt, and Result types
@@ -39,6 +48,7 @@ pub const RUN_AGAIN: RunAgain = true;
 pub const DONT_RUN_AGAIN: RunAgain = false;
 
 #[doc(hidden)]
+#[cfg(feature = "code")]
 error_chain! {
     types {
         Error, ErrorKind, ResultExt, Result;

flowr/src/lib/coordinator.rs

@@ -444,25 +444,27 @@ impl Coordinator {
         server_context: Arc<Mutex<RuntimeServerConnection>>,
         native: bool,
     ) -> Result<Manifest> {
-        let native_url =
+        let flowruntimelib_url =
             Url::parse("lib://flowruntime").chain_err(|| "Could not parse lib_manifest_url")?;
 
         // Load this run-time's library of native (statically linked) implementations
         loader
             .add_lib(
                 provider,
                 flowruntime::get_manifest(server_context)?,
-                &native_url,
+                &flowruntimelib_url,
             )
             .chain_err(|| "Could not add 'flowruntime' library to loader")?;
 
         // If the "native" feature is enabled then load the native flowstdlib if command line arg to do so
         if cfg!(feature = "native") && native {
+            let flowstdlib_url =
+                Url::parse("lib://flowstdlib").chain_err(|| "Could not parse flowstdlib_url")?;
             loader
                 .add_lib(
                     provider,
                     flowstdlib::get_manifest().chain_err(|| "Could not get flowstdlib manifest")?,
-                    &native_url,
+                    &flowstdlib_url,
                 )
                 .chain_err(|| "Could not add 'flowstdlib' library to loader")?;
         }

flowstdlib/control/compare_switch/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 
 serde_json = "1.0"
 

flowstdlib/control/index/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 serde_json = "1.0"
 
 [workspace]

flowstdlib/control/join/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 
 serde_json = "1.0"
 

flowstdlib/control/route/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 serde_json = "1.0"
 
 [workspace]

flowstdlib/control/select/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 serde_json = "1.0"
 
 [workspace]

flowstdlib/control/tap/flow.toml

@@ -15,8 +15,8 @@ lto = true
 opt-level = 's'
 
 [dependencies]
-flow_impl = "=0.10.0"
-flow_impl_derive = "=0.10.0"
+flowcore = {path = "../../../flowcore", version = "=0.34.7", default-features = false}
+flow_impl_derive = "=0.34.7"
 serde_json = "1.0"
 
 [workspace]