Conformance tests

.github/workflows/rust.yml

@@ -9,6 +9,9 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
+permissions:
+  pull-requests: write
+
 jobs:
   test:
     name: Test
@@ -24,3 +27,38 @@ jobs:
         run: cargo clippy --all-features --all-targets -- -D warnings
       - name: Run tests
         run: cargo test --verbose
+  conformance:
+    name: Conformance Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+      - name: Run conformance tests
+        run: cargo test --test conformance 2>&1 | tee conformance.log
+        continue-on-error: true
+      - id: conformance
+        name: Conformance results
+        run: |
+          read -r passed skipped failed <<< $(cat conformance.log | perl -n -e'/steps \((\d+) passed, (\d+) skipped, (\d+) failed\)/ && print "$1 $2 $3"')
+
+          echo "passed=$passed"
+          echo "skipped=$skipped"
+          echo "failed=$failed"
+
+          # Calculate the conformance score as a percentage (passed / (passed + skipped + failed))
+          score=$(echo "scale=2; $passed / ($passed + $skipped + $failed) * 100" | bc)
+
+          # Set the output
+          echo "score=$score" >> "$GITHUB_OUTPUT"
+          echo "passed=$passed" >> "$GITHUB_OUTPUT"
+          echo "skipped=$skipped" >> "$GITHUB_OUTPUT"
+          echo "failed=$failed" >> "$GITHUB_OUTPUT"
+      - name: Comment PR
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            **Conformance score**: ${{ steps.conformance.outputs.score }}%
+            **Passed**: ${{ steps.conformance.outputs.passed }}
+            **Skipped**: ${{ steps.conformance.outputs.skipped }}
+            **Failed**: ${{ steps.conformance.outputs.failed }}
+

interpreter/Cargo.toml

@@ -3,7 +3,10 @@ name = "cel-interpreter"
 description = "An interpreter for the Common Expression Language (CEL)"
 repository = "https://github.com/clarkmcc/cel-rust"
 version = "0.8.1"
-authors = ["Tom Forbes <[email protected]>", "Clark McCauley <[email protected]>"]
+authors = [
+    "Tom Forbes <[email protected]>",
+    "Clark McCauley <[email protected]>",
+]
 edition = "2021"
 license = "MIT"
 categories = ["compilers"]
@@ -20,7 +23,13 @@ regex = "1.10.5"
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 serde_bytes = "0.11.14"
+cucumber = "0.20"
+futures = "0.3"
 
 [[bench]]
 name = "runtime"
 harness = false
+
+[[test]]
+name = "conformance"
+harness = false

interpreter/src/context.rs

@@ -3,6 +3,7 @@ use crate::objects::{TryIntoValue, Value};
 use crate::{functions, ExecutionError};
 use cel_parser::Expression;
 use std::collections::HashMap;
+use std::fmt::{Debug, Formatter};
 
 /// Context is a collection of variables and functions that can be used
 /// by the interpreter to resolve expressions. The context can be either
@@ -39,6 +40,12 @@ pub enum Context<'a> {
     },
 }
 
+impl<'a> Debug for Context<'a> {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Context {{ ... }}")
+    }
+}
+
 impl<'a> Context<'a> {
     pub fn add_variable<S, V>(
         &mut self,

interpreter/src/gherkin.rs

@@ -0,0 +1,124 @@
+use crate::objects::Map;
+use crate::Value;
+use nom::branch::alt;
+use nom::bytes::complete::tag;
+use nom::character::complete::{char, digit1};
+use nom::combinator::{map_res, opt, recognize};
+use nom::sequence::{delimited, preceded};
+use nom::IResult;
+use std::str::FromStr;
+use std::sync::Arc;
+
+pub fn parse_value(input: &str) -> Result<Value, ()> {
+    let (_, v) = _parse_value(input).map_err(|err| {
+        eprintln!("Error parsing value: {:?}", err);
+        ()
+    })?;
+    Ok(v)
+}
+
+fn _parse_value(input: &str) -> IResult<&str, Value> {
+    alt((
+        parse_none,
+        parse_bytes,
+        parse_double,
+        parse_int,
+        parse_uint,
+        parse_string,
+        parse_bool,
+        parse_list,
+        parse_map,
+    ))(input)
+}
+
+fn parse_int(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("IntType(source=")(input)?;
+    let (i, v) = map_res(recognize(preceded(opt(char('-')), digit1)), |s| {
+        i64::from_str(s)
+    })(i)?;
+    Ok((i, Value::Int(v)))
+}
+
+fn parse_uint(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("UintType(source=")(input)?;
+    let (i, v) = map_res(recognize(digit1), u64::from_str)(i)?;
+    Ok((i, Value::UInt(v)))
+}
+
+fn parse_double(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("DoubleType(source=")(input)?;
+    let (i, v) = map_res(recognize(preceded(opt(char('-')), digit1)), |s| {
+        f64::from_str(s)
+    })(i)?;
+    Ok((i, Value::Float(v)))
+}
+
+fn parse_string(i: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("StringType(source=")(i)?;
+    let (i, value_str) = delimited(
+        alt((char('"'), char('\''))),
+        alt((tag("!"), nom::character::complete::alphanumeric0)),
+        alt((char('"'), char('\''))),
+    )(i)?;
+    Ok((i, Value::String(Arc::new(value_str.to_string()))))
+}
+
+fn parse_bytes(i: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("BytesType(source=b")(i)?;
+    let (i, value_str) = delimited(
+        char('\''),
+        nom::character::complete::alphanumeric0,
+        char('\''),
+    )(i)?;
+    Ok((i, Value::Bytes(value_str.as_bytes().to_vec().into())))
+}
+
+fn parse_bool(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("BoolType(source=")(input)?;
+    let (i, value_str) = alt((tag("False"), tag("True")))(i)?;
+    let value = match value_str {
+        "False" => false,
+        "True" => true,
+        _ => unreachable!(),
+    };
+    Ok((i, Value::Bool(value)))
+}
+
+fn parse_list(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("[]")(input)?; // only empty list supported for now
+    Ok((i, Value::List(vec![].into())))
+}
+
+fn parse_map(input: &str) -> IResult<&str, Value> {
+    let (i, _) = tag("MapType({})")(input)?; // only empty map supported for now
+    Ok((i, Value::Map(Map::default())))
+}
+
+fn parse_none(input: &str) -> IResult<&str, Value> {
+    let (input, _) = tag("None")(input)?;
+    Ok((input, Value::Null))
+}
+
+#[test]
+fn test_parse_value_type() {
+    let value = parse_value("IntType(source=123)").unwrap();
+    assert_eq!(value, Value::Int(123));
+}
+
+#[test]
+fn test_parse_int_type() {
+    let (_, value) = parse_int("IntType(source=-9223372036854775808)").unwrap();
+    assert_eq!(value, Value::Int(-9223372036854775808));
+}
+
+#[test]
+fn test_parse_string_type() {
+    let (_, value) = parse_string("StringType(source='!')").unwrap();
+    assert_eq!(value, Value::String(Arc::new("!".to_string())));
+}
+
+#[test]
+fn test_parse_bool_type() {
+    let (_, value) = parse_bool("BoolType(source=False)").unwrap();
+    assert_eq!(value, Value::Bool(false));
+}

interpreter/src/lib.rs

@@ -5,6 +5,8 @@ use std::convert::TryFrom;
 use std::sync::Arc;
 use thiserror::Error;
 
+pub mod gherkin;
+
 mod macros;
 
 pub mod context;

interpreter/src/objects.rs

@@ -13,7 +13,7 @@ use std::convert::{Infallible, TryFrom, TryInto};
 use std::fmt::{Display, Formatter};
 use std::sync::Arc;
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Default, Debug, PartialEq, Clone)]
 pub struct Map {
     pub map: Arc<HashMap<Key, Value>>,
 }

interpreter/src/ser.rs

@@ -568,8 +568,32 @@ mod tests {
     use serde_bytes::Bytes;
     use std::{collections::HashMap, iter::FromIterator, sync::Arc};
 
+    macro_rules! primitive_test {
+        ($functionName:ident, $strValue: literal, $value: expr) => {
+            #[test]
+            fn $functionName() {
+                let program = Program::compile($strValue).unwrap();
+                let result = program.execute(&Context::default());
+                assert_eq!(Value::from($value), result.unwrap());
+            }
+        };
+    }
+
+    primitive_test!(test_u64_zero, "0u", 0_u64);
+    primitive_test!(test_i64_zero, "0", 0_i64);
+    primitive_test!(test_f64_zero, "0.0", 0_f64);
+    //primitive_test!(test_f64_zero, "0.", 0_f64); this test fails
+    primitive_test!(test_bool_false, "false", false);
+    primitive_test!(test_bool_true, "true", true);
+    primitive_test!(test_string_empty, "\"\"", "");
+    primitive_test!(test_string_non_empty, "\"test\"", "test");
+    primitive_test!(test_byte_ones, r#"b"\001\001""#, vec!(1_u8, 1_u8));
+    // primitive_test!(test_triple_double_quoted_string, #"r"""""""#, "");
+    // primitive_test!(test_triple_single_quoted_string, "r''''''", "");
+    primitive_test!(test_utf8_character_as_bytes, "b'ÿ'", vec!(195_u8, 191_u8));
+
     #[test]
-    fn test_primitives() {
+    fn test_json_data_conversion() {
         #[derive(Serialize)]
         struct TestPrimitives {
             bool: bool,

interpreter/tests/conformance.rs

@@ -0,0 +1,39 @@
+use cel_interpreter::{Context, Program};
+use cucumber::{given, then, when, World};
+
+// `World` is your shared, likely mutable state.
+// Cucumber constructs it via `Default::default()` for each scenario.
+#[derive(Debug, Default, World)]
+pub struct CelWorld {
+    expression: String,
+    context: Context<'static>,
+}
+
+#[when(expr = "CEL expression \"{word}\" is evaluated")]
+fn expression_evaluated_with_double_quotes(world: &mut CelWorld, expression: String) {
+    world.expression = expression;
+}
+
+#[when(expr = "CEL expression '{word}\' is evaluated")]
+fn expression_evaluated_with_single_quotes(world: &mut CelWorld, expression: String) {
+    world.expression = expression;
+}
+
+// Given: bindings parameter "x" is IntType(source=123)
+#[given(regex = "^bindings parameter \"([a-z]+)\" is (.*)$")]
+fn bindings_parameter_is(world: &mut CelWorld, name: String, value: String) {
+    println!("bindings_parameter_is: name={}, value={}", name, value);
+    // world.context.add_variable(name, value);
+}
+
+#[then(regex = "value is (.*)")]
+fn evaluation_result_is(world: &mut CelWorld, expected_result: String) {
+    let program = Program::compile(&world.expression).unwrap();
+    let result = program.execute(&world.context);
+    let expect = cel_interpreter::gherkin::parse_value(&expected_result).unwrap();
+    assert_eq!(expect, result.unwrap());
+}
+
+fn main() {
+    futures::executor::block_on(CelWorld::run("tests/features"));
+}