add digest(utf8, method) function

ballista/rust/core/proto/ballista.proto

@@ -153,6 +153,7 @@ enum ScalarFunction {
   SHA512 = 33;
   LN = 34;
   TOTIMESTAMPMILLIS = 35;
+  DIGEST = 36;
 }
 
 message ScalarFunctionNode {

ballista/rust/core/src/serde/logical_plan/from_proto.rs

@@ -27,8 +27,8 @@ use datafusion::logical_plan::window_frames::{
     WindowFrame, WindowFrameBound, WindowFrameUnits,
 };
 use datafusion::logical_plan::{
-    abs, acos, asin, atan, ceil, cos, exp, floor, ln, log10, log2, round, signum, sin,
-    sqrt, tan, trunc, Column, DFField, DFSchema, Expr, JoinConstraint, JoinType,
+    abs, acos, asin, atan, ceil, cos, digest, exp, floor, ln, log10, log2, round, signum,
+    sin, sqrt, tan, trunc, Column, DFField, DFSchema, Expr, JoinConstraint, JoinType,
     LogicalPlan, LogicalPlanBuilder, Operator,
 };
 use datafusion::physical_plan::aggregates::AggregateFunction;
@@ -1152,6 +1152,9 @@ impl TryInto<Expr> for &protobuf::LogicalExprNode {
                     protobuf::ScalarFunction::Sha512 => {
                         Ok(sha512((&args[0]).try_into()?))
                     }
+                    protobuf::ScalarFunction::Digest => {
+                        Ok(digest((&args[0]).try_into()?, (&args[1]).try_into()?))
+                    }
                     _ => Err(proto_error(
                         "Protobuf deserialization error: Unsupported scalar function",
                     )),

ballista/rust/core/src/serde/logical_plan/to_proto.rs

@@ -1485,6 +1485,7 @@ impl TryInto<protobuf::ScalarFunction> for &BuiltinScalarFunction {
             BuiltinScalarFunction::SHA256 => Ok(protobuf::ScalarFunction::Sha256),
             BuiltinScalarFunction::SHA384 => Ok(protobuf::ScalarFunction::Sha384),
             BuiltinScalarFunction::SHA512 => Ok(protobuf::ScalarFunction::Sha512),
+            BuiltinScalarFunction::Digest => Ok(protobuf::ScalarFunction::Digest),
             BuiltinScalarFunction::ToTimestampMillis => {
                 Ok(protobuf::ScalarFunction::Totimestampmillis)
             }

ballista/rust/core/src/serde/physical_plan/from_proto.rs

@@ -559,6 +559,7 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction {
             ScalarFunction::Sha256 => BuiltinScalarFunction::SHA256,
             ScalarFunction::Sha384 => BuiltinScalarFunction::SHA384,
             ScalarFunction::Sha512 => BuiltinScalarFunction::SHA512,
+            ScalarFunction::Digest => BuiltinScalarFunction::Digest,
             ScalarFunction::Ln => BuiltinScalarFunction::Ln,
             ScalarFunction::Totimestampmillis => BuiltinScalarFunction::ToTimestampMillis,
         }

datafusion/src/logical_plan/expr.rs

@@ -1508,7 +1508,7 @@ macro_rules! unary_scalar_expr {
     };
 }
 
-/// Create an convenience function representing a /binaryunary scalar function
+/// Create an convenience function representing a binary scalar function
 macro_rules! binary_scalar_expr {
     ($ENUM:ident, $FUNC:ident) => {
         #[doc = "this scalar function is not documented yet"]
@@ -1581,6 +1581,7 @@ unary_scalar_expr!(Upper, upper);
 // date functions
 binary_scalar_expr!(DatePart, date_part);
 binary_scalar_expr!(DateTrunc, date_trunc);
+binary_scalar_expr!(Digest, digest);
 
 /// returns an array of fixed size with each argument on it.
 pub fn array(args: Vec<Expr>) -> Expr {
@@ -2171,6 +2172,7 @@ mod tests {
         test_unary_scalar_expr!(SHA256, sha256);
         test_unary_scalar_expr!(SHA384, sha384);
         test_unary_scalar_expr!(SHA512, sha512);
+        test_unary_scalar_expr!(Digest, digest);
         test_unary_scalar_expr!(SplitPart, split_part);
         test_unary_scalar_expr!(StartsWith, starts_with);
         test_unary_scalar_expr!(Strpos, strpos);

datafusion/src/logical_plan/mod.rs

@@ -38,14 +38,15 @@ pub use display::display_schema;
 pub use expr::{
     abs, acos, and, array, ascii, asin, atan, avg, binary_expr, bit_length, btrim, case,
     ceil, character_length, chr, col, columnize_expr, combine_filters, concat, concat_ws,
-    cos, count, count_distinct, create_udaf, create_udf, date_part, date_trunc, exp,
-    exprlist_to_fields, floor, in_list, initcap, left, length, lit, lit_timestamp_nano,
-    ln, log10, log2, lower, lpad, ltrim, max, md5, min, normalize_col, normalize_cols,
-    now, octet_length, or, random, regexp_match, regexp_replace, repeat, replace,
-    replace_col, reverse, right, round, rpad, rtrim, sha224, sha256, sha384, sha512,
-    signum, sin, split_part, sqrt, starts_with, strpos, substr, sum, tan, to_hex,
-    translate, trim, trunc, unnormalize_col, unnormalize_cols, upper, when, Column, Expr,
-    ExprRewriter, ExpressionVisitor, Literal, Recursion, RewriteRecursion,
+    cos, count, count_distinct, create_udaf, create_udf, date_part, date_trunc, digest,
+    exp, exprlist_to_fields, floor, in_list, initcap, left, length, lit,
+    lit_timestamp_nano, ln, log10, log2, lower, lpad, ltrim, max, md5, min,
+    normalize_col, normalize_cols, now, octet_length, or, random, regexp_match,
+    regexp_replace, repeat, replace, replace_col, reverse, right, round, rpad, rtrim,
+    sha224, sha256, sha384, sha512, signum, sin, split_part, sqrt, starts_with, strpos,
+    substr, sum, tan, to_hex, translate, trim, trunc, unnormalize_col, unnormalize_cols,
+    upper, when, Column, Expr, ExprRewriter, ExpressionVisitor, Literal, Recursion,
+    RewriteRecursion,
 };
 pub use extension::UserDefinedLogicalNode;
 pub use operators::Operator;