feat: Lex JSON Pointers in Policy Expressions

This change is the initial step towards integrating JSON Pointers
as a native part of the Policy Expression language.

This should have no visible effect on the behavior of Hipcheck yet,
as it's not possible for the lexer to read JSON Pointer syntax after
the preprocessor strips them out.

hipcheck/src/policy_exprs/token.rs

@@ -36,6 +36,9 @@ pub enum Token {
 
 	#[regex("([a-zA-Z]+)", lex_ident)]
 	Ident(String),
+
+	#[regex(r"\$[/~_[:alnum:]]*", lex_json_pointer)]
+	JSONPointer(String),
 }
 
 /// Lex a single boolean.
@@ -70,6 +73,20 @@ fn lex_ident(input: &mut Lexer<'_, Token>) -> Result<String> {
 	Ok(input.slice().to_owned())
 }
 
+/// Lex a JSON Pointer.
+fn lex_json_pointer(input: &mut Lexer<'_, Token>) -> Result<String> {
+	// Remove the initial '$' character.
+	// Slice safety: the slice is guaranteed to be at least 1 character long,
+	// because the regex for JSONPointer contains a mandatory '$' character.
+	let s = &input.slice()[1..];
+	if let Some(chr) = s.chars().next() {
+		if chr != '/' {
+			return Err(LexingError::InvalidJSONPointer(s.to_owned()));
+		}
+	}
+	Ok(s.to_owned())
+}
+
 impl Display for Token {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		match self {
@@ -82,6 +99,7 @@ impl Display for Token {
 			Token::Integer(i) => write!(f, "{i}"),
 			Token::Float(fl) => write!(f, "{fl}"),
 			Token::Ident(i) => write!(f, "{i}"),
+			Token::JSONPointer(pointer) => write!(f, "${pointer}"),
 		}
 	}
 }
@@ -104,11 +122,14 @@ pub enum LexingError {
 
 	#[error("invalid boolean, found '{0}'")]
 	InvalidBool(String),
+
+	#[error("invalid JSON Pointer, found '{0}'. JSON Pointers must be empty or start with '/'.")]
+	InvalidJSONPointer(String),
 }
 
 #[cfg(test)]
 mod tests {
-	use crate::policy_exprs::{token::Token, Result, F64};
+	use crate::policy_exprs::{token::Token, Error::Lex, LexingError, Result, F64};
 	use logos::Logos as _;
 	use test_log::test;
 
@@ -161,4 +182,46 @@ mod tests {
 		let tokens = lex(raw_program).unwrap();
 		assert_eq!(tokens, expected);
 	}
+
+	#[test]
+	fn basic_lexing_with_jsonptr_empty() {
+		let raw_program = "$";
+		let expected = vec![
+			// Note that the initial '$' is not considered part of the pointer string.
+			Token::JSONPointer(String::from("")),
+		];
+		let tokens = lex(raw_program).unwrap();
+		assert_eq!(tokens, expected);
+	}
+
+	#[test]
+	fn basic_lexing_with_jsonptr_error_invalid() {
+		// This JSON Pointer is invalid because it doesn't start with a '/' character.
+		let raw_program = "$alpha";
+		let expected = Err(Lex(LexingError::InvalidJSONPointer(String::from("alpha"))));
+		let tokens = lex(raw_program);
+		assert_eq!(tokens, expected);
+	}
+
+	#[test]
+	fn basic_lexing_with_jsonptr_valid_chars() {
+		let raw_program = "$/alpha_bravo/~0/~1";
+		let expected = vec![Token::JSONPointer(String::from("/alpha_bravo/~0/~1"))];
+		let tokens = lex(raw_program).unwrap();
+		assert_eq!(tokens, expected);
+	}
+
+	#[test]
+	fn basic_lexing_with_jsonptr_in_expr() {
+		let raw_program = "(eq 1 $/data/one)";
+		let expected = vec![
+			Token::OpenParen,
+			Token::Ident(String::from("eq")),
+			Token::Integer(1),
+			Token::JSONPointer(String::from("/data/one")),
+			Token::CloseParen,
+		];
+		let tokens = lex(raw_program).unwrap();
+		assert_eq!(tokens, expected);
+	}
 }