From b6ea2bfab7591858fb5d47d853043e739e48c496 Mon Sep 17 00:00:00 2001 From: Alexander Weber Date: Fri, 21 Jul 2023 17:51:23 -0400 Subject: [PATCH] 0.5 --- .github/workflows/wiki.yml | 2 +- .neomake.yaml | 10 +- Cargo.lock | 64 +++++++++ Cargo.toml | 1 + res/templates/max.yaml | 46 ++++--- res/templates/python.yaml | 18 +-- src/args.rs | 2 +- src/error.rs | 2 + src/main.rs | 3 +- src/model.rs | 262 ++++++++++++++++++++++--------------- src/plan.rs | 60 +++++++++ src/workflow.rs | 112 ++++++++++++++-- test/.neomake.yaml | 98 +++++++++++--- 13 files changed, 509 insertions(+), 171 deletions(-) create mode 100644 src/plan.rs diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index 9fd8e1c..5b1b652 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -22,7 +22,7 @@ jobs: cargo run --target=x86_64-unknown-linux-gnu -- man -o ./markdown -f markdown # render and include schema - cargo run --target=x86_64-unknown-linux-gnu -- config schema > ./docs/schema.json + cargo run --target=x86_64-unknown-linux-gnu -- workflow schema > ./docs/schema.json fmerge merge -p "<-- %f -->" -f ./docs/README.md > ./docs/README_MERGED.md mv ./docs/README_MERGED.md ./docs/README.md diff --git a/.neomake.yaml b/.neomake.yaml index 7d399b5..b7502fd 100644 --- a/.neomake.yaml +++ b/.neomake.yaml @@ -3,10 +3,12 @@ version: "0.5" nodes: build: matrix: - - - env: - RELEASE: "" - - env: - RELEASE: "--release" + dense: + dimensions: + - - env: + RELEASE: "" + - env: + RELEASE: "--release" tasks: - script: | set -e diff --git a/Cargo.lock b/Cargo.lock index 452c9ab..049a487 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +dependencies = [ + "memchr", +] + [[package]] name = "async-trait" version = "0.1.51" @@ -25,6 +34,21 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -230,6 +254,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fsio" version = "0.1.3" @@ -436,6 +470,7 @@ dependencies = [ "clap_complete", "clap_mangen", "crossterm", + "fancy-regex", "handlebars", "interactive_process", "itertools", @@ -579,6 +614,35 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + [[package]] name = "roff" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index f74d4e7..f297141 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ threadpool = "^1.8" schemars = "0.8.12" toml = "0.7.6" ron = "0.8.0" +fancy-regex = "0.11.0" [dev-dependencies] rusty-hook = "0.11.2" diff --git a/res/templates/max.yaml b/res/templates/max.yaml index 4f6a531..0f17127 100644 --- a/res/templates/max.yaml +++ b/res/templates/max.yaml @@ -16,10 +16,12 @@ nodes: args: - -c matrix: - - - env: - PRINT_VAL: value 0 - - env: - PRINT_VAL: value 1 + dense: + dimensions: + - - env: + PRINT_VAL: value 0 + - env: + PRINT_VAL: value 1 tasks: - shell: program: python @@ -31,20 +33,22 @@ nodes: a: matrix: - - - env: - VA: A0 - - env: - VA: A1 - - - env: - VB: B0 - - env: - VB: B1 - - env: - VB: B2 - - - env: - VC: C0 - - env: - VC: C1 + dense: + dimensions: + - - env: + VA: A0 + - env: + VA: A1 + - - env: + VB: B0 + - env: + VB: B1 + - env: + VB: B2 + - - env: + VC: C0 + - env: + VC: C1 tasks: - script: echo "$VA $VB $VC" b: @@ -75,8 +79,10 @@ nodes: test: matrix: - - - env: - OVERRIDE_ENV_VAR_0: new e0 + dense: + dimensions: + - - env: + OVERRIDE_ENV_VAR_0: new e0 tasks: - env: OVERRIDE_ENV_VAR_1: new e1 diff --git a/res/templates/python.yaml b/res/templates/python.yaml index 891de05..516c069 100644 --- a/res/templates/python.yaml +++ b/res/templates/python.yaml @@ -3,14 +3,16 @@ version: "0.5" nodes: count: matrix: - - - env: - X: "0" - - env: - X: "1" - - - env: - Y: "0" - - env: - Y: "1" + dense: + dimensions: + - - env: + X: "0" + - env: + X: "1" + - - env: + Y: "0" + - env: + Y: "1" tasks: - shell: program: "python3" diff --git a/src/args.rs b/src/args.rs index 0fef01e..3339e63 100644 --- a/src/args.rs +++ b/src/args.rs @@ -8,7 +8,7 @@ use std::{ use clap::{Arg, ArgAction}; -use crate::{error::Error, model::ExecutionPlan}; +use crate::{error::Error, plan::ExecutionPlan}; #[derive(Debug, Eq, PartialEq)] pub(crate) enum Privilege { diff --git a/src/error.rs b/src/error.rs index e0ec879..1c914b5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -41,4 +41,6 @@ pub(crate) enum Error { DeserializeRon(#[from] ron::error::SpannedError), #[error("handlebars")] Handlebars(#[from] handlebars::RenderError), + #[error("regex")] + RegexError(#[from] fancy_regex::Error), } diff --git a/src/main.rs b/src/main.rs index 71ce437..f9fec95 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ pub mod args; pub mod error; pub mod model; pub mod output; +pub mod plan; pub mod reference; pub mod workflow; @@ -84,7 +85,7 @@ async fn main() -> Result<(), crate::error::Error> { format, } => { let m = model::Workflow::load(&workflow)?; - m.plan(&nodes, &format).await?; + m.describe(&nodes, &format).await?; Ok(()) }, } diff --git a/src/model.rs b/src/model.rs index a91f313..09cf224 100644 --- a/src/model.rs +++ b/src/model.rs @@ -11,36 +11,13 @@ use threadpool::ThreadPool; use crate::{ error::Error, output::{self, Controller}, - workflow::Shell, + plan, }; -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct Task { - cmd: String, - env: HashMap, - workdir: Option, - shell: Shell, -} -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct Matrix { - tasks: Vec, -} -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct Node { - matrix: Vec, -} -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct Stage { - nodes: Vec, -} -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub(crate) struct ExecutionPlan { - stages: Vec, -} - pub(crate) struct Workflow { pub nodes: HashMap, pub env: HashMap, + pub capture: Option, } impl Workflow { @@ -62,6 +39,7 @@ impl Workflow { let cfg: crate::workflow::Workflow = serde_yaml::from_str(&data)?; Ok(Self { nodes: cfg.nodes, + capture: cfg.capture, env: if let Some(e) = cfg.env { e } else { @@ -74,78 +52,103 @@ impl Workflow { &self, nodes: &HashSet, args: &HashMap, - ) -> Result { + ) -> Result { let mut hb = handlebars::Handlebars::new(); hb.set_strict_mode(true); let arg_vals = self.build_args(args)?; let stages = self.determine_order(nodes)?; - let mut plan = ExecutionPlan { stages: vec![] }; + let mut plan = plan::ExecutionPlan { + stages: vec![], + nodes: HashMap::<_, _>::new(), + env: self.env.clone(), + }; + // captured env variables + if let Some(v) = &self.capture { + let regex = fancy_regex::Regex::new(v)?; + let envs = std::env::vars().collect_vec(); + for e in envs { + if regex.is_match(&e.0)? { + plan.env.insert(e.0, e.1); + } + } + } for stage in stages { - let mut rendered_stage = Stage { nodes: vec![] }; + let mut rendered_stage = plan::Stage { nodes: vec![] }; for node in stage { - let mut rendered_node = Node { matrix: vec![] }; // nodes + tasks -> parallelize on l0 let node_def = &self.nodes[&node]; - - let matrix_entry_default = crate::workflow::MatrixCell { ..Default::default() }; - - let matrix_cp = if let Some(matrix) = &node_def.matrix { - matrix.iter().multi_cartesian_product().collect::>() - } else { - vec![vec![&matrix_entry_default]] + let mut rendered_node = plan::Node { + invocations: vec![], + tasks: vec![], + + env: HashMap::<_, _>::new(), + shell: match node_def.shell.clone() { + | Some(v) => Some(v.into()), + | None => None, + }, + workdir: node_def.workdir.clone(), }; - - for mat in matrix_cp { - let mut rendered_matrix = Matrix { tasks: vec![] }; - for task in &node_def.tasks { - let rendered_cmd = hb.render_template(&task.script, &arg_vals)?; - - let workdir = if let Some(workdir) = &task.workdir { - Some(workdir.to_owned()) - } else if let Some(workdir) = &node_def.workdir { - Some(workdir.to_owned()) - } else { - None - }; - - let shell = if let Some(shell) = &task.shell { - shell.to_owned() - } else if let Some(shell) = &node_def.shell { - shell.to_owned() - } else { - crate::workflow::Shell { - program: "sh".to_owned(), - args: vec!["-c".to_owned()], - } - }; - - let mut rendered_task = Task { - cmd: rendered_cmd, - env: HashMap::::new(), - workdir, - shell, - }; - - let mut combined_matrix_env = Some(HashMap::::new()); - for i in 0..mat.len() { - if let Some(env_current) = &mat[i].env { - combined_matrix_env.as_mut().unwrap().extend(env_current.clone()); - } + // captured env variables + if let Some(v) = &node_def.capture { + let regex = fancy_regex::Regex::new(v)?; + let envs = std::env::vars().collect_vec(); + for e in envs { + if regex.is_match(&e.0)? { + rendered_node.env.insert(e.0, e.1); } - - let self_env = Some(self.env.clone()); - for env in vec![&self_env, &node_def.env, &combined_matrix_env, &task.env] { - if let Some(m) = env { - rendered_task.env.extend(m.clone()); - } - } - - rendered_matrix.tasks.push(rendered_task); } - rendered_node.matrix.push(rendered_matrix); } - rendered_stage.nodes.push(rendered_node); + // explicitly set vars override + rendered_node.env.extend(match node_def.env.clone() { + | Some(v) => v, + | None => HashMap::<_, _>::new(), + }); + + // default to one matrix entry + let invocation_default = vec![crate::plan::Invocation { ..Default::default() }]; + + for task in &node_def.tasks { + let rendered_cmd = hb.render_template(&task.script, &arg_vals)?; + + rendered_node.tasks.push(plan::Task { + cmd: rendered_cmd, + shell: match task.shell.clone() { + | Some(v) => Some(v.into()), + | None => None, + }, + env: match task.env.clone() { + | Some(v) => v, + | None => HashMap::<_, _>::new(), + }, + workdir: task.workdir.clone(), + }); + } + + rendered_node.invocations = match &node_def.matrix { + | Some(m) => m.compile()?, + | None => invocation_default, + }; + + // rendered_node.invocations = match &node_def.matrix { + // | Some(v) => v + // .iter() + // .multi_cartesian_product() + // .map(|x| { + // let mut env = HashMap::::new(); + // for i in x { + // if let Some(e) = &i.env { + // env.extend(e.clone()); + // } + // } + // plan::Invocation { env } + // }) + // .collect_vec(), + // | None => matrix_entry_default, + // }; + + plan.nodes.insert(node.clone(), rendered_node); + rendered_stage.nodes.push(node); } plan.stages.push(rendered_stage); } @@ -181,7 +184,7 @@ impl Workflow { Ok(()) } - pub async fn plan(&self, nodes: &HashSet, format: &crate::args::Format) -> Result<(), Error> { + pub async fn describe(&self, nodes: &HashSet, format: &crate::args::Format) -> Result<(), Error> { let structure = self.determine_order(&nodes)?; #[derive(Debug, serde::Serialize)] @@ -307,30 +310,74 @@ impl ExecEngine { } } - pub async fn execute(&self, plan: ExecutionPlan, workers: usize) -> Result<(), Error> { - for stage in plan.stages { - let signal_cnt = stage.nodes.iter().map(|c| c.matrix.len()).sum(); + pub async fn execute(&self, plan: plan::ExecutionPlan, workers: usize) -> Result<(), Error> { + struct Work { + workdir: Option, + env: HashMap, + shell: plan::Shell, + command: String, + } + + for stage in &plan.stages { let pool = ThreadPool::new(workers); - let (tx, rx) = std::sync::mpsc::channel::>(); + let (signal_tx, signal_rx) = std::sync::mpsc::channel::>(); + let mut signal_cnt = 0; + + let nodes = stage.nodes.iter().map(|v| plan.nodes.get(v).unwrap()); + for node in nodes { + for matrix in &node.invocations { + let t_tx = signal_tx.clone(); + let t_output = self.output.clone(); + + let mut work = Vec::::new(); + + for task in &node.tasks { + let workdir = if let Some(workdir) = &task.workdir { + Some(workdir.to_owned()) + } else if let Some(workdir) = &node.workdir { + Some(workdir.to_owned()) + } else { + None + }; - for node in stage.nodes { - for matrix in node.matrix { - let output_thread = self.output.clone(); - let tx_thread = tx.clone(); + let shell = if let Some(shell) = &task.shell { + shell.to_owned() + } else if let Some(shell) = &node.shell { + shell.to_owned() + } else { + crate::plan::Shell { + program: "sh".to_owned(), + args: vec!["-c".to_owned()], + } + }; + + let mut env = plan.env.clone(); + env.extend(node.env.clone()); + env.extend(matrix.env.clone()); + env.extend(task.env.clone()); + + signal_cnt += 1; + work.push(Work { + command: task.cmd.clone(), + env, + shell, + workdir, + }) + } // executes matrix entry pool.execute(move || { let res = move || -> Result<(), Box> { - for task in matrix.tasks { - let mut cmd_proc = std::process::Command::new(&task.shell.program); - cmd_proc.args(task.shell.args); - cmd_proc.envs(task.env); - if let Some(w) = task.workdir { + for w in work { + let mut cmd_proc = std::process::Command::new(w.shell.program); + cmd_proc.args(w.shell.args); + cmd_proc.envs(w.env); + if let Some(w) = w.workdir { cmd_proc.current_dir(w); } - cmd_proc.arg(&task.cmd); + cmd_proc.arg(&w.command); - let loc_out = output_thread.clone(); + let loc_out = t_output.clone(); let exit_status = InteractiveProcess::new(cmd_proc, move |l| match l { | Ok(v) => { let mut lock = loc_out.lock().unwrap(); @@ -341,7 +388,7 @@ impl ExecEngine { .wait()?; if let Some(code) = exit_status.code() { if code != 0 { - let err_msg = format!("command \"{}\" failed with code {}", &task.cmd, code); + let err_msg = format!("command \"{}\" failed with code {}", &w.command, code); return Err(Box::new(Error::ChildProcess(err_msg))); } } @@ -349,16 +396,17 @@ impl ExecEngine { Ok(()) }(); match res { - | Ok(..) => tx_thread.send(Ok(())).expect("send failed"), - | Err(e) => tx_thread - // error formatting should be improved - .send(Err(Error::Generic(format!("{:?}", e)))) - .expect("send failed"), + | Ok(..) => t_tx.send(Ok(())).expect("send failed"), + | Err(e) => t_tx + // error formatting should be improved + .send(Err(Error::Generic(format!("{:?}", e)))) + .expect("send failed"), } }); } } - let errs = rx + + let errs = signal_rx .iter() .take(signal_cnt) .filter(|x| x.is_err()) diff --git a/src/plan.rs b/src/plan.rs new file mode 100644 index 0000000..614683f --- /dev/null +++ b/src/plan.rs @@ -0,0 +1,60 @@ +use std::collections::HashMap; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct ExecutionPlan { + pub nodes: HashMap, + pub stages: Vec, + + pub env: HashMap, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct Stage { + pub nodes: Vec, +} + +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct Shell { + pub program: String, + pub args: Vec, +} + +impl From for Shell { + fn from(value: crate::workflow::Shell) -> Self { + Self { + program: value.program.clone(), + args: value.args.clone(), + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct Node { + pub invocations: Vec, + pub tasks: Vec, + + pub env: HashMap, + pub shell: Option, + pub workdir: Option, +} + +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct Invocation { + pub coords: String, + pub env: HashMap, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub(crate) struct Task { + pub cmd: String, + + pub env: HashMap, + pub shell: Option, + pub workdir: Option, +} diff --git a/src/workflow.rs b/src/workflow.rs index 32a91ef..4f4f202 100644 --- a/src/workflow.rs +++ b/src/workflow.rs @@ -1,43 +1,135 @@ use std::collections::HashMap; +use itertools::Itertools; + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] #[serde(rename_all = "snake_case")] // can not deny unknown fields to support YAML anchors +/// The entire workflow definition. pub(crate) struct Workflow { + /// The version of this workflow file (major.minor). pub version: String, + /// RegEx for capturing env vars at plan time + baking these into the plan. + pub capture: Option, + /// Explicitly set env vars. pub env: Option>, + /// All nodes. pub nodes: HashMap, } +#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +/// A task execution environment. +pub(crate) struct Shell { + /// The program (like "/bin/bash"). + pub program: String, + /// Custom args (like \["-c"\]). + pub args: Vec, +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] #[serde(rename_all = "snake_case", deny_unknown_fields)] +/// An individual node for executing a task batch. pub(crate) struct Node { + /// A description of this node. pub description: Option, - pub env: Option>, - pub workdir: Option, + /// Reference nodes that need to be executed prior to this one. pub pre: Option>, - pub matrix: Option>>, + + /// An n-dimensional matrix that is executed for every item in its cartesian product. + pub matrix: Option, + /// The tasks to be executed. pub tasks: Vec, + + /// RegEx for capturing env vars at plan time + baking these into the plan. + pub capture: Option, + /// Explicitly set env vars. + pub env: Option>, + /// Custom program to execute the scripts. pub shell: Option, + /// Custom workdir. + pub workdir: Option, } -#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] #[serde(rename_all = "snake_case", deny_unknown_fields)] -pub(crate) struct Shell { - pub program: String, - pub args: Vec, +/// An entry in the n-dimensional matrix for the node execution. +pub(crate) enum Matrix { + Dense { + drop: Option, + dimensions: Vec>, + }, + Sparse { + dimensions: Vec>, + keep: Option, + }, } -#[derive(Debug, Default, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] +impl Matrix { + pub(crate) fn compile(&self) -> Result, crate::error::Error> { + let (dimensions, regex, invert) = match self { + | Self::Dense { drop, dimensions } => (dimensions, drop, true), + | Self::Sparse { keep, dimensions } => (dimensions, keep, false), + }; + + let regex = fancy_regex::Regex::new(match regex { + | Some(s) => s, + | None => ".*", + })?; + + // Bake the coords in their respective dimension into the struct itself. + // This makes coord finding for regex (later) a breeze. + let dims_widx = dimensions.iter().map(|d_x| { + let mut y = 0usize; + d_x.iter() + .map(|d_y| { + y += 1; + (y - 1, d_y) + }) + .collect_vec() + }); + + let cp = dims_widx.multi_cartesian_product(); + let mut v = Vec::::new(); + + for next in cp { + let coords = next.iter().map(|v| format!("{}", v.0)).join(","); + + // Use the baked-in coords to invoke regex. + if regex.is_match(&format!("{}", coords))? ^ !invert { + continue; + } + + let mut env = HashMap::::new(); + for m in next { + if let Some(e) = &m.1.env { + env.extend(e.clone()); + } + } + + v.push(crate::plan::Invocation { env, coords }); + } + Ok(v) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] #[serde(rename_all = "snake_case", deny_unknown_fields)] +/// An entry in the n-dimensional matrix for the node execution. pub(crate) struct MatrixCell { pub env: Option>, } #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, schemars::JsonSchema)] #[serde(rename_all = "snake_case", deny_unknown_fields)] +/// An individual task. pub(crate) struct Task { - pub workdir: Option, + /// The script content to execute. Can contain handlebars placeholders. + pub script: String, + + /// Explicitly set env vars. pub env: Option>, + /// Custom program to execute the scripts. pub shell: Option, - pub script: String, + /// Custom workdir. + pub workdir: Option, } diff --git a/test/.neomake.yaml b/test/.neomake.yaml index da9b1cc..d329ce3 100644 --- a/test/.neomake.yaml +++ b/test/.neomake.yaml @@ -1,19 +1,29 @@ version: "0.5" +capture: "^(PWD)$" env: ENV_GLOBAL: "val_global" nodes: count: matrix: - - - env: - X: "0" - - env: - X: "1" - - - env: - Y: "0" - - env: - Y: "1" + dense: + drop: "^(0,0,0|1,1,1)$" + # sparse: + # keep: "^(0,0,0|1,1,1)$" + dimensions: + - - env: + X: "0" + - env: + X: "1" + - - env: + Y: "0" + - env: + Y: "1" + - - env: + Z: "0" + - env: + Z: "1" tasks: - shell: program: "python3" @@ -26,10 +36,26 @@ nodes: cnt = 0 while True: - print(f'{os.environ.get("X")}{os.environ.get("Y")}: {cnt}', flush=True) + print(f'{os.environ.get("X")}{os.environ.get("Y")}{os.environ.get("Z")}: {cnt}', flush=True) cnt+=1 time.sleep(random.random()) + print: + matrix: + dense: + dimensions: + - - env: + X: "0" + - env: + X: "1" + - - env: + Y: "0" + - env: + Y: "1" + tasks: + - script: | + printf $X-$Y + alpha: # minimal example tasks: - script: | @@ -46,8 +72,10 @@ nodes: env: ENV_NODE: val_NODE matrix: - - - env: - ENV_MATRIX: val_matrix + dense: + dimensions: + - - env: + ENV_MATRIX: "val_matrix" tasks: - env: ENV_TASK: val_task @@ -60,14 +88,16 @@ nodes: delta: # matrix matrix: - - - env: - X: "0" - - env: - X: "1" - - - env: - Y: "0" - - env: - Y: "1" + dense: + dimensions: + - - env: + X: "0" + - env: + X: "1" + - - env: + Y: "0" + - env: + Y: "1" tasks: - script: | set -e @@ -100,3 +130,33 @@ nodes: script: | set -e printf "$0" + golf: + tasks: + - script: | + set -e + exit 1 + + l0_0: + pre: [] + capture: "^(HOME)$" + env: + HOME: some-home + tasks: + - script: printf l0_0 + l1_0: + pre: + - l0_0 + tasks: + - script: printf l1_0 + l1_1: + capture: "^(HOME)$" + pre: + - l0_0 + tasks: + - script: printf l1_1 + l2_0: + pre: + - l1_0 + - l1_1 + tasks: + - script: printf l2_0