x-pack/filebeat/input/cel: use structpb.Struct as intermediate type (#…

…35915)

This replaces the conversion to a structpb.Struct-serialisation-deserialisation
path that was being used to a direct conversion to a structpb.Struct type which
can then be used to obtain a map[string]any directly.

Using the test suite as a set of benchmarks shows good performance improvements
almost across the board, with no regression. See #35139 for details
of the benchmarks. Note that the benchmark results shown here are at that commit
or that commit plus the change here. No substantive change has been made to the
package to invalidate the comparison.

goos: darwin
goarch: amd64
pkg: github.com/elastic/beats/v7/x-pack/filebeat/input/cel
cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
                                                                        │   cel.bench   │             cel-new.bench             │
                                                                        │    sec/op     │    sec/op      vs base                │
Input/hello_world-16                                                       54.08µ ±  2%   37.92µ ±   3%  -29.88% (p=0.000 n=10)
Input/bad_events_type-16                                                   51.45µ ±  3%   37.13µ ±   2%  -27.84% (p=0.000 n=10)
Input/hello_world_non_nil_state-16                                         53.21µ ±  1%   37.11µ ±   3%  -30.27% (p=0.000 n=10)
Input/what_is_next-16                                                      62.86µ ± 14%   38.48µ ±   3%  -38.79% (p=0.000 n=10)
Input/bad_cursor_type-16                                                   59.79µ ±  1%   40.40µ ±   3%  -32.42% (p=0.000 n=10)
Input/show_state-16                                                        53.69µ ±  3%   38.87µ ±   3%  -27.60% (p=0.000 n=10)
Input/show_provided_state-16                                               65.13µ ±  3%   43.54µ ±   2%  -33.15% (p=0.000 n=10)
Input/iterative_state-16                                                    2.000 ±  0%    2.001 ±   0%        ~ (p=0.247 n=10)
Input/iterative_state_implicit_initial_cursor-16                            2.001 ±  0%    2.000 ±   0%        ~ (p=0.063 n=10)
Input/iterative_state_provided_stored_cursor-16                             1.001 ±  0%    1.000 ±   0%   -0.09% (p=0.029 n=10)
Input/iterative_state_implicit_initial_cursor_provided_stored_cursor-16     1.001 ±  0%    1.001 ±   0%        ~ (p=0.912 n=10)
Input/strings_split-16                                                     69.23µ ±  3%   48.67µ ±   4%  -29.70% (p=0.000 n=10)
Input/ndjson_log_file_simple-16                                            164.7µ ±  3%   147.1µ ±   3%  -10.70% (p=0.000 n=10)
Input/ndjson_log_file_simple_file_scheme-16                                167.0µ ±  6%   152.4µ ±   6%   -8.77% (p=0.000 n=10)
Input/ndjson_log_file_corrupted-16                                         179.3µ ±  6%   153.2µ ±   5%  -14.58% (p=0.000 n=10)
Input/missing_file-16                                                      67.98µ ±  2%   52.31µ ±   3%  -23.05% (p=0.000 n=10)
Input/decode_xml-16                                                        871.6µ ± 28%   780.2µ ±  25%        ~ (p=0.143 n=10)
Input/GET_request-16                                                       728.8µ ± 73%   690.1µ ±  70%        ~ (p=0.579 n=10)
Input/retry_after_request-16                                               766.5µ ± 58%   623.6µ ± 134%        ~ (p=0.393 n=10)
Input/retry_after_request_time-16                                          1.277m ± 31%   1.044m ±  14%        ~ (p=0.247 n=10)
Input/rate_limit_request_0-16                                              1.143m ± 45%   1.059m ±   6%        ~ (p=0.123 n=10)
Input/rate_limit_request_10-16                                             1.089m ± 35%   1.067m ±   7%        ~ (p=0.631 n=10)
Input/rate_limit_request_10_too_slow-16                                    994.7µ ± 29%   930.8µ ±  36%        ~ (p=0.481 n=10)
Input/retry_failure-16                                                    1184.4µ ± 51%   959.6µ ±  12%        ~ (p=0.143 n=10)
Input/POST_request-16                                                      1.376m ± 39%   1.125m ±  21%        ~ (p=0.393 n=10)
Input/repeated_POST_request-16                                             110.1m ±  4%   117.2m ±   2%   +6.39% (p=0.002 n=10)
Input/split_events-16                                                      877.6µ ± 30%   793.4µ ±  35%        ~ (p=0.631 n=10)
Input/split_events_keep_parent-16                                          1.148m ± 54%   1.048m ±  21%        ~ (p=0.529 n=10)
Input/nested_split_events-16                                               1.124m ±  6%   1.038m ±   7%   -7.68% (p=0.009 n=10)
Input/absent_split-16                                                       4.999 ±  0%    4.999 ±   0%        ~ (p=0.684 n=10)
Input/date_cursor-16                                                        2.002 ±  0%    2.002 ±   0%        ~ (p=0.912 n=10)
Input/tracer_filename_sanitization-16                                       2.002 ±  0%    2.002 ±   0%        ~ (p=0.218 n=10)
Input/pagination_cursor_object-16                                           1.001 ±  0%    1.002 ±   0%   +0.09% (p=0.000 n=10)
Input/pagination_cursor_array-16                                            1.002 ±  0%    1.002 ±   0%   +0.08% (p=0.023 n=10)
Input/first_event_cursor-16                                                 3.002 ±  0%    3.001 ±   0%        ~ (p=0.529 n=10)
Input/OAuth2-16                                                            1.243m ± 28%   1.082m ±  23%        ~ (p=0.165 n=10)
Input/simple_multistep_GET_request-16                                      2.821m ± 57%   2.291m ± 107%        ~ (p=0.529 n=10)
Input/three_step_GET_request-16                                            2.821m ± 23%   2.996m ±  32%        ~ (p=0.853 n=10)
Input/type_error_message-16                                               1463.4µ ± 61%   969.2µ ±  89%        ~ (p=0.280 n=10)
geomean                                                                    3.751m         3.243m         -13.56%

                                                                        │   cel.bench    │             cel-new.bench              │
                                                                        │      B/op      │      B/op       vs base                │
Input/hello_world-16                                                       27.76Ki ±  0%    25.97Ki ±  0%   -6.46% (p=0.000 n=10)
Input/bad_events_type-16                                                   26.01Ki ±  0%    24.74Ki ±  0%   -4.91% (p=0.000 n=10)
Input/hello_world_non_nil_state-16                                         27.47Ki ±  0%    25.67Ki ±  0%   -6.54% (p=0.000 n=10)
Input/what_is_next-16                                                      30.49Ki ±  0%    27.89Ki ±  0%   -8.53% (p=0.000 n=10)
Input/bad_cursor_type-16                                                   28.84Ki ±  0%    26.65Ki ±  0%   -7.58% (p=0.000 n=10)
Input/show_state-16                                                        27.24Ki ±  0%    25.53Ki ±  0%   -6.30% (p=0.000 n=10)
Input/show_provided_state-16                                               28.76Ki ±  0%    26.34Ki ±  0%   -8.41% (p=0.000 n=10)
Input/iterative_state-16                                                   55.05Ki ±  1%    45.29Ki ±  1%  -17.74% (p=0.000 n=10)
Input/iterative_state_implicit_initial_cursor-16                           59.17Ki ±  0%    49.39Ki ±  0%  -16.53% (p=0.000 n=10)
Input/iterative_state_provided_stored_cursor-16                            45.98Ki ±  1%    38.63Ki ±  0%  -15.97% (p=0.000 n=10)
Input/iterative_state_implicit_initial_cursor_provided_stored_cursor-16    49.48Ki ±  1%    42.13Ki ±  0%  -14.86% (p=0.000 n=10)
Input/strings_split-16                                                     33.30Ki ±  0%    30.48Ki ±  0%   -8.46% (p=0.000 n=10)
Input/ndjson_log_file_simple-16                                            36.40Ki ±  0%    33.69Ki ±  0%   -7.45% (p=0.000 n=10)
Input/ndjson_log_file_simple_file_scheme-16                                36.54Ki ±  0%    33.83Ki ±  0%   -7.42% (p=0.000 n=10)
Input/ndjson_log_file_corrupted-16                                         39.92Ki ±  0%    36.24Ki ±  0%   -9.20% (p=0.000 n=10)
Input/missing_file-16                                                      27.92Ki ±  0%    26.10Ki ±  0%   -6.52% (p=0.000 n=10)
Input/decode_xml-16                                                        89.81Ki ±  0%    82.66Ki ±  0%   -7.96% (p=0.000 n=10)
Input/GET_request-16                                                       60.89Ki ±  0%    57.45Ki ±  0%   -5.66% (p=0.000 n=10)
Input/retry_after_request-16                                               59.99Ki ±  0%    56.13Ki ±  0%   -6.43% (p=0.001 n=10)
Input/retry_after_request_time-16                                          60.00Ki ±  0%    56.19Ki ±  0%   -6.35% (p=0.000 n=10)
Input/rate_limit_request_0-16                                              63.69Ki ±  0%    58.42Ki ±  0%   -8.27% (p=0.000 n=10)
Input/rate_limit_request_10-16                                             63.64Ki ±  0%    58.54Ki ±  0%   -8.02% (p=0.000 n=10)
Input/rate_limit_request_10_too_slow-16                                    63.77Ki ±  0%    58.45Ki ±  0%   -8.34% (p=0.001 n=10)
Input/retry_failure-16                                                     60.12Ki ±  0%    56.31Ki ±  0%   -6.34% (p=0.001 n=10)
Input/POST_request-16                                                      64.29Ki ±  0%    60.64Ki ±  0%   -5.67% (p=0.000 n=10)
Input/repeated_POST_request-16                                            103.90Ki ±  2%    97.28Ki ±  3%   -6.37% (p=0.000 n=10)
Input/split_events-16                                                      59.25Ki ±  0%    56.50Ki ±  0%   -4.63% (p=0.000 n=10)
Input/split_events_keep_parent-16                                          64.27Ki ±  0%    60.44Ki ±  0%   -5.96% (p=0.000 n=10)
Input/nested_split_events-16                                               62.33Ki ±  0%    59.42Ki ±  0%   -4.68% (p=0.000 n=10)
Input/absent_split-16                                                      175.5Ki ±  3%    172.8Ki ±  4%        ~ (p=0.089 n=10)
Input/date_cursor-16                                                       134.7Ki ±  6%    136.6Ki ±  7%        ~ (p=0.393 n=10)
Input/tracer_filename_sanitization-16                                      209.5Ki ±  5%    212.0Ki ±  4%        ~ (p=0.684 n=10)
Input/pagination_cursor_object-16                                          99.86Ki ± 12%   100.96Ki ±  8%        ~ (p=0.739 n=10)
Input/pagination_cursor_array-16                                           98.43Ki ± 16%   103.33Ki ± 12%        ~ (p=0.912 n=10)
Input/first_event_cursor-16                                                155.1Ki ± 14%    151.9Ki ± 19%        ~ (p=0.353 n=10)
Input/OAuth2-16                                                            81.98Ki ±  0%    79.54Ki ±  0%   -2.97% (p=0.000 n=10)
Input/simple_multistep_GET_request-16                                      83.07Ki ±  0%    78.70Ki ±  1%   -5.26% (p=0.000 n=10)
Input/three_step_GET_request-16                                            105.3Ki ±  0%    100.7Ki ±  1%   -4.35% (p=0.000 n=10)
Input/type_error_message-16                                                50.86Ki ±  0%    50.24Ki ±  0%   -1.24% (p=0.000 n=10)
geomean                                                                    58.17Ki          54.43Ki         -6.43%

                                                                        │  cel.bench   │            cel-new.bench             │
                                                                        │  allocs/op   │  allocs/op    vs base                │
Input/hello_world-16                                                       204.0 ±  0%    166.0 ±  1%  -18.63% (p=0.000 n=10)
Input/bad_events_type-16                                                   177.0 ±  0%    151.0 ±  0%  -14.69% (p=0.000 n=10)
Input/hello_world_non_nil_state-16                                         202.0 ±  0%    164.0 ±  0%  -18.81% (p=0.000 n=10)
Input/what_is_next-16                                                      261.0 ±  0%    203.0 ±  0%  -22.22% (p=0.000 n=10)
Input/bad_cursor_type-16                                                   235.0 ±  0%    187.0 ±  0%  -20.43% (p=0.000 n=10)
Input/show_state-16                                                        199.0 ±  0%    164.0 ±  0%  -17.59% (p=0.000 n=10)
Input/show_provided_state-16                                               269.0 ±  0%    204.0 ±  0%  -24.16% (p=0.000 n=10)
Input/iterative_state-16                                                   702.0 ±  1%    485.0 ±  1%  -30.91% (p=0.000 n=10)
Input/iterative_state_implicit_initial_cursor-16                           766.0 ±  0%    549.0 ±  0%  -28.33% (p=0.000 n=10)
Input/iterative_state_provided_stored_cursor-16                            507.5 ±  1%    361.0 ±  1%  -28.87% (p=0.000 n=10)
Input/iterative_state_implicit_initial_cursor_provided_stored_cursor-16    553.0 ±  1%    406.0 ±  1%  -26.58% (p=0.000 n=10)
Input/strings_split-16                                                     306.0 ±  0%    243.0 ±  0%  -20.59% (p=0.000 n=10)
Input/ndjson_log_file_simple-16                                            319.0 ±  0%    254.0 ±  0%  -20.38% (p=0.000 n=10)
Input/ndjson_log_file_simple_file_scheme-16                                319.0 ±  0%    254.0 ±  0%  -20.38% (p=0.000 n=10)
Input/ndjson_log_file_corrupted-16                                         373.0 ±  0%    294.0 ±  0%  -21.18% (p=0.000 n=10)
Input/missing_file-16                                                      204.0 ±  0%    168.0 ±  0%  -17.65% (p=0.000 n=10)
Input/decode_xml-16                                                        903.0 ±  0%    749.0 ±  0%  -17.05% (p=0.000 n=10)
Input/GET_request-16                                                       540.0 ±  0%    467.0 ±  0%  -13.52% (p=0.000 n=10)
Input/retry_after_request-16                                               570.0 ±  0%    479.0 ±  0%  -15.96% (p=0.000 n=10)
Input/retry_after_request_time-16                                          570.0 ±  0%    479.0 ±  0%  -15.96% (p=0.000 n=10)
Input/rate_limit_request_0-16                                              623.0 ±  0%    511.0 ±  0%  -17.98% (p=0.000 n=10)
Input/rate_limit_request_10-16                                             623.0 ±  0%    512.0 ±  0%  -17.82% (p=0.000 n=10)
Input/rate_limit_request_10_too_slow-16                                    623.0 ±  0%    512.0 ±  0%  -17.82% (p=0.000 n=10)
Input/retry_failure-16                                                     571.0 ±  0%    479.0 ±  0%  -16.11% (p=0.000 n=10)
Input/POST_request-16                                                      579.0 ±  0%    497.0 ±  0%  -14.16% (p=0.000 n=10)
Input/repeated_POST_request-16                                            1045.5 ±  1%    879.5 ±  0%  -15.88% (p=0.000 n=10)
Input/split_events-16                                                      504.0 ±  0%    442.0 ±  0%  -12.30% (p=0.000 n=10)
Input/split_events_keep_parent-16                                          581.0 ±  0%    500.0 ±  0%  -13.94% (p=0.000 n=10)
Input/nested_split_events-16                                               559.0 ±  0%    498.0 ±  0%  -10.91% (p=0.000 n=10)
Input/absent_split-16                                                     1.625k ±  1%   1.448k ±  1%  -10.92% (p=0.000 n=10)
Input/date_cursor-16                                                      1.353k ±  1%   1.358k ±  1%        ~ (p=0.566 n=10)
Input/tracer_filename_sanitization-16                                     1.697k ±  1%   1.690k ±  1%        ~ (p=0.566 n=10)
Input/pagination_cursor_object-16                                          869.0 ± 44%    874.5 ± 30%        ~ (p=0.493 n=10)
Input/pagination_cursor_array-16                                           870.5 ± 50%    869.5 ± 37%        ~ (p=1.000 n=10)
Input/first_event_cursor-16                                               1.603k ±  1%   1.596k ±  2%        ~ (p=0.541 n=10)
Input/OAuth2-16                                                            658.5 ±  0%    621.0 ±  0%   -5.69% (p=0.000 n=10)
Input/simple_multistep_GET_request-16                                      740.0 ±  0%    665.0 ±  0%  -10.14% (p=0.000 n=10)
Input/three_step_GET_request-16                                            939.0 ±  0%    864.0 ±  0%   -7.99% (p=0.000 n=10)
Input/type_error_message-16                                                342.0 ±  0%    341.0 ±  0%   -0.29% (p=0.000 n=10)
geomean                                                                    526.0          444.9        -15.42%

CHANGELOG.next.asciidoc

@@ -327,6 +327,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Add Okta input package for entity analytics. {pull}35611[35611]
 - Expose harvester metrics from filestream input {pull}35835[35835] {issue}33771[33771]
 - Add device support for Azure AD entity analytics. {pull}35807[35807]
+- Improve CEL input performance. {pull}35915[35915]
 
 *Auditbeat*
    - Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817]

x-pack/filebeat/input/cel/input.go

@@ -10,7 +10,6 @@ package cel
 import (
 	"compress/gzip"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -33,8 +32,6 @@ import (
 
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/checker/decls"
-	"google.golang.org/protobuf/encoding/protojson"
-	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/structpb"
 
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
@@ -909,23 +906,20 @@ func evalWith(ctx context.Context, prg cel.Program, state map[string]interface{}
 		return state, fmt.Errorf("failed eval: %w", err)
 	}
 
-	v, err := out.ConvertToNative(reflect.TypeOf(&structpb.Value{}))
+	v, err := out.ConvertToNative(reflect.TypeOf((*structpb.Struct)(nil)))
 	if err != nil {
 		state["events"] = errorMessage(fmt.Sprintf("failed proto conversion: %v", err))
 		return state, fmt.Errorf("failed proto conversion: %w", err)
 	}
-	b, err := protojson.MarshalOptions{Indent: ""}.Marshal(v.(proto.Message))
-	if err != nil {
-		state["events"] = errorMessage(fmt.Sprintf("failed native conversion: %v", err))
-		return state, fmt.Errorf("failed native conversion: %w", err)
-	}
-	var res map[string]interface{}
-	err = json.Unmarshal(b, &res)
-	if err != nil {
-		state["events"] = errorMessage(fmt.Sprintf("failed json conversion: %v", err))
-		return state, fmt.Errorf("failed json conversion: %w", err)
+	switch v := v.(type) {
+	case *structpb.Struct:
+		return v.AsMap(), nil
+	default:
+		// This should never happen.
+		errMsg := fmt.Sprintf("unexpected native conversion type: %T", v)
+		state["events"] = errorMessage(errMsg)
+		return state, errors.New(errMsg)
 	}
-	return res, nil
 }
 
 func errorMessage(msg string) map[string]interface{} {