From fea3e6452e603fa3deb58570c2c9b883bb5196a8 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Wed, 13 Jan 2021 06:41:26 -0700 Subject: [PATCH 001/769] fix(external docs): Update component metrics (#6010) * Apply default metrics only to transforms Signed-off-by: Luc Perkins * Update first batch of components Signed-off-by: Luc Perkins * Finish remaining components Signed-off-by: Luc Perkins --- docs/reference/components.cue | 10 +++--- docs/reference/components/kafka.cue | 2 ++ .../components/sinks/aws_kinesis_streams.cue | 5 +++ docs/reference/components/sinks/aws_sqs.cue | 6 ++++ docs/reference/components/sinks/blackhole.cue | 5 +++ docs/reference/components/sinks/console.cue | 2 ++ docs/reference/components/sinks/nats.cue | 7 ++++ .../components/sinks/sematext_metrics.cue | 2 +- docs/reference/components/sinks/socket.cue | 6 ++++ .../reference/components/sinks/splunk_hec.cue | 10 +++--- docs/reference/components/sinks/vector.cue | 2 ++ .../components/sources/apache_metrics.cue | 10 ++++++ .../components/sources/aws_ecs_metrics.cue | 10 ++++++ .../components/sources/docker_logs.cue | 4 ++- .../components/sources/generator.cue | 4 +++ .../components/sources/host_metrics.cue | 4 +++ .../components/sources/internal_metrics.cue | 36 +++++++++++++++++++ .../reference/components/sources/journald.cue | 2 ++ .../components/sources/kubernetes_logs.cue | 2 ++ .../components/sources/mongodb_metrics.cue | 7 ++++ .../components/sources/nginx_metrics.cue | 7 ++++ .../sources/prometheus_remote_write.cue | 19 ++++++---- docs/reference/components/sources/socket.cue | 7 +++- docs/reference/components/sources/statsd.cue | 2 ++ docs/reference/components/sources/stdin.cue | 2 ++ docs/reference/components/sources/syslog.cue | 2 ++ .../components/transforms/add_fields.cue | 4 +++ .../components/transforms/coercer.cue | 4 +++ .../components/transforms/concat.cue | 4 +++ .../components/transforms/filter.cue | 8 ++--- .../reference/components/transforms/geoip.cue | 4 +++ .../components/transforms/json_parser.cue | 4 +++ .../transforms/key_value_parser.cue | 4 +++ .../components/transforms/log_to_metric.cue | 4 +++ 34 files changed, 189 insertions(+), 22 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 7cfbc6854dc8c..08c43287cd86f 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -1046,10 +1046,12 @@ components: { } } - telemetry: metrics: { - // Default metrics for each component - processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total - processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + if Kind == "transform" { + telemetry: metrics: { + // Default metrics for each transform + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + } } }} } diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue index fb76249bc4745..7a191207e9e19 100644 --- a/docs/reference/components/kafka.cue +++ b/docs/reference/components/kafka.cue @@ -102,5 +102,7 @@ components: _kafka: { telemetry: metrics: { consumer_offset_updates_failed_total: components.sources.internal_metrics.output.metrics.consumer_offset_updates_failed_total events_failed_total: components.sources.internal_metrics.output.metrics.events_failed_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total } } diff --git a/docs/reference/components/sinks/aws_kinesis_streams.cue b/docs/reference/components/sinks/aws_kinesis_streams.cue index 6c96f6c124fe2..45514a3af243a 100644 --- a/docs/reference/components/sinks/aws_kinesis_streams.cue +++ b/docs/reference/components/sinks/aws_kinesis_streams.cue @@ -163,4 +163,9 @@ components: sinks: aws_kinesis_streams: components._aws & { ] }, ] + + telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + } } diff --git a/docs/reference/components/sinks/aws_sqs.cue b/docs/reference/components/sinks/aws_sqs.cue index 87ab4981bfcf0..96c652961df89 100644 --- a/docs/reference/components/sinks/aws_sqs.cue +++ b/docs/reference/components/sinks/aws_sqs.cue @@ -111,4 +111,10 @@ components: sinks: aws_sqs: components._aws & { ] }, ] + + telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total + } } diff --git a/docs/reference/components/sinks/blackhole.cue b/docs/reference/components/sinks/blackhole.cue index 4b948a05234ce..4417de18ee6f2 100644 --- a/docs/reference/components/sinks/blackhole.cue +++ b/docs/reference/components/sinks/blackhole.cue @@ -55,4 +55,9 @@ components: sinks: blackhole: { logs: true metrics: null } + + telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + } } diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue index 109e91814471a..737d50ba27af5 100644 --- a/docs/reference/components/sinks/console.cue +++ b/docs/reference/components/sinks/console.cue @@ -77,6 +77,8 @@ components: sinks: console: { } telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/nats.cue b/docs/reference/components/sinks/nats.cue index 86bc735d88931..d72e62e601b66 100644 --- a/docs/reference/components/sinks/nats.cue +++ b/docs/reference/components/sinks/nats.cue @@ -88,4 +88,11 @@ components: sinks: nats: { logs: true metrics: null } + + telemetry: metrics: { + missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + send_errors_total: components.sources.internal_metrics.output.metrics.send_errors_total + } } diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue index 58e94e23bbf2d..276b4ebbc5913 100644 --- a/docs/reference/components/sinks/sematext_metrics.cue +++ b/docs/reference/components/sinks/sematext_metrics.cue @@ -79,7 +79,7 @@ components: sinks: sematext_metrics: { } telemetry: metrics: { - processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total encode_errors_total: components.sources.internal_metrics.output.metrics.encode_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/socket.cue b/docs/reference/components/sinks/socket.cue index c23bba653b574..6938815535939 100644 --- a/docs/reference/components/sinks/socket.cue +++ b/docs/reference/components/sinks/socket.cue @@ -99,4 +99,10 @@ components: sinks: socket: { logs: true metrics: null } + + telemetry: metrics: { + connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index c012ee7840afc..98d18ce455f7e 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -160,9 +160,11 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - encode_errors_total: components.sources.internal_metrics.output.metrics.encode_errors_total - missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total - processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total - processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + encode_errors_total: components.sources.internal_metrics.output.metrics.encode_errors_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + requests_received_total: components.sources.internal_metrics.output.metrics.requests_received_total } } diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue index 61e2344705173..ba8a0c02f8ad5 100644 --- a/docs/reference/components/sinks/vector.cue +++ b/docs/reference/components/sinks/vector.cue @@ -88,6 +88,8 @@ components: sinks: vector: { how_it_works: components.sources.vector.how_it_works telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total } } diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue index 5cd303bbc23ea..b3125288654b0 100644 --- a/docs/reference/components/sources/apache_metrics.cue +++ b/docs/reference/components/sources/apache_metrics.cue @@ -191,4 +191,14 @@ components: sources: apache_metrics: { } how_it_works: {} + + telemetry: metrics: { + http_error_response_total: components.sources.internal_metrics.output.metrics.http_error_response_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + requests_completed_total: components.sources.internal_metrics.output.metrics.requests_completed_total + request_duration_nanoseconds: components.sources.internal_metrics.output.metrics.request_duration_nanoseconds + } } diff --git a/docs/reference/components/sources/aws_ecs_metrics.cue b/docs/reference/components/sources/aws_ecs_metrics.cue index 1d8f04759697d..eb2397d517f54 100644 --- a/docs/reference/components/sources/aws_ecs_metrics.cue +++ b/docs/reference/components/sources/aws_ecs_metrics.cue @@ -237,4 +237,14 @@ components: sources: aws_ecs_metrics: { network_transmit_packets_drop_total: _awsecs & _network_counter & {description: "Number of outbound packets dropped by the container."} network_transmit_errs_total: _awsecs & _network_counter & {description: "Errors sending packets."} } + + telemetry: metrics: { + http_error_response_total: components.sources.internal_metrics.output.metrics.http_error_response_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + requests_completed_total: components.sources.internal_metrics.output.metrics.requests_completed_total + request_duration_nanoseconds: components.sources.internal_metrics.output.metrics.request_duration_nanoseconds + } } diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index 05631ca4fc4e4..104292b9b1c83 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -340,10 +340,12 @@ components: sources: docker_logs: { telemetry: metrics: { communication_errors_total: components.sources.internal_metrics.output.metrics.communication_errors_total - container_processed_events_total: components.sources.internal_metrics.output.metrics.container_processed_events_total container_metadata_fetch_errors_total: components.sources.internal_metrics.output.metrics.container_metadata_fetch_errors_total + container_processed_events_total: components.sources.internal_metrics.output.metrics.container_processed_events_total containers_unwatched_total: components.sources.internal_metrics.output.metrics.containers_unwatched_total containers_watched_total: components.sources.internal_metrics.output.metrics.containers_watched_total logging_driver_errors_total: components.sources.internal_metrics.output.metrics.logging_driver_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total } } diff --git a/docs/reference/components/sources/generator.cue b/docs/reference/components/sources/generator.cue index 2f7294fe364e0..b4c60bd390d82 100644 --- a/docs/reference/components/sources/generator.cue +++ b/docs/reference/components/sources/generator.cue @@ -100,4 +100,8 @@ components: sources: generator: { } output: logs: {} + + telemetry: metrics: { + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + } } diff --git a/docs/reference/components/sources/host_metrics.cue b/docs/reference/components/sources/host_metrics.cue index 6642286601729..cb2c4abdff8f7 100644 --- a/docs/reference/components/sources/host_metrics.cue +++ b/docs/reference/components/sources/host_metrics.cue @@ -406,4 +406,8 @@ components: sources: host_metrics: { } _network_nomac: _network_gauge & {relevant_when: "OS is not macOS"} } + + telemetry: metrics: { + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + } } diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue index 9be884faa4e23..2e9cde3f42936 100644 --- a/docs/reference/components/sources/internal_metrics.cue +++ b/docs/reference/components/sources/internal_metrics.cue @@ -77,6 +77,30 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _internal_metrics_tags } + connection_established_total: { + description: "The total number of times a connection has been established." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + connection_failed_total: { + description: "The total number of times a connection has failed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + connection_send_errors_total: { + description: "The total number of errors sending data via the connection." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + connection_shutdown_total: { + description: "The total number of times the connection has been shut down." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } quit_total: { description: "The total number of times the Vector instance has quit." type: "counter" @@ -429,6 +453,12 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _internal_metrics_tags } + request_duration_nanoseconds: { + description: "The total request duration in nanoseconds." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } request_read_errors_total: { description: "The total number of request read errors for this component." type: "counter" @@ -447,6 +477,12 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _component_tags } + send_errors_total: { + description: "The total number of errors sending messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } sqs_message_delete_failed_total: { description: "The total number of failures to delete SQS messages." type: "counter" diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index 61440db7a3a9c..460282d765ac8 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -194,5 +194,7 @@ components: sources: journald: { telemetry: metrics: { invalid_record_total: components.sources.internal_metrics.output.metrics.invalid_record_total invalid_record_bytes_total: components.sources.internal_metrics.output.metrics.invalid_record_bytes_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total } } diff --git a/docs/reference/components/sources/kubernetes_logs.cue b/docs/reference/components/sources/kubernetes_logs.cue index 8684cf10f10bc..fb449ed98c726 100644 --- a/docs/reference/components/sources/kubernetes_logs.cue +++ b/docs/reference/components/sources/kubernetes_logs.cue @@ -510,5 +510,7 @@ components: sources: kubernetes_logs: { k8s_format_picker_edge_cases_total: components.sources.internal_metrics.output.metrics.k8s_format_picker_edge_cases_total k8s_docker_format_parse_failures_total: components.sources.internal_metrics.output.metrics.k8s_docker_format_parse_failures_total k8s_event_annotation_failures_total: components.sources.internal_metrics.output.metrics.k8s_event_annotation_failures_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 0d989d79e7abf..fd5c1bd2ce152 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -750,4 +750,11 @@ components: sources: mongodb_metrics: { tags: _mongodb_metrics_tags } } + + telemetry: metrics: { + collect_completed_total: components.sources.internal_metrics.output.metrics.collect_completed_total + collect_duration_nanoseconds: components.sources.internal_metrics.output.metrics.collect_duration_nanoseconds + parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total + request_errors_total: components.sources.internal_metrics.output.metrics.request_errors_total + } } diff --git a/docs/reference/components/sources/nginx_metrics.cue b/docs/reference/components/sources/nginx_metrics.cue index 112f1179de510..948ff172f9be5 100644 --- a/docs/reference/components/sources/nginx_metrics.cue +++ b/docs/reference/components/sources/nginx_metrics.cue @@ -172,4 +172,11 @@ components: sources: nginx_metrics: { tags: _nginx_metrics_tags } } + + telemetry: metrics: { + collect_completed_total: components.sources.internal_metrics.output.metrics.collect_completed_total + collect_duration_nanoseconds: components.sources.internal_metrics.output.metrics.collect_duration_nanoseconds + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total + } } diff --git a/docs/reference/components/sources/prometheus_remote_write.cue b/docs/reference/components/sources/prometheus_remote_write.cue index 9af190c408e3f..2118cca585f3b 100644 --- a/docs/reference/components/sources/prometheus_remote_write.cue +++ b/docs/reference/components/sources/prometheus_remote_write.cue @@ -70,12 +70,6 @@ components: sources: prometheus_remote_write: { gauge: output._passthrough_gauge } - telemetry: metrics: { - parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total - processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total - processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total - } - how_it_works: { metric_types: { title: "Metric type interpretation" @@ -83,7 +77,7 @@ components: sources: prometheus_remote_write: { The remote_write protocol used by this source transmits only the metric tags, timestamp, and numerical value. No explicit information about the original type of the - metric (ie counter, histogram, etc) is included. As + metric (i.e. counter, histogram, etc) is included. As such, this source makes a guess as to what the original metric type was. @@ -93,4 +87,15 @@ components: sources: prometheus_remote_write: { """ } } + + telemetry: metrics: { + http_error_response_total: components.sources.internal_metrics.output.metrics.http_error_response_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + parse_errors_total: components.sources.internal_metrics.output.metrics.parse_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total + requests_completed_total: components.sources.internal_metrics.output.metrics.requests_completed_total + requests_received_total: components.sources.internal_metrics.output.metrics.requests_received_total + request_duration_nanoseconds: components.sources.internal_metrics.output.metrics.request_duration_nanoseconds + } } diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index 56454dd65611b..3b8bf65cff832 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -152,6 +152,11 @@ components: sources: socket: { ] telemetry: metrics: { - connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total + connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total + connection_failed_total: components.sources.internal_metrics.output.metrics.connection_failed_total + connection_established_total: components.sources.internal_metrics.output.metrics.connection_established_total + connection_failed_total: components.sources.internal_metrics.output.metrics.connection_failed_total + connection_send_errors_total: components.sources.internal_metrics.output.metrics.connection_send_errors_total + connection_shutdown_total: components.sources.internal_metrics.output.metrics.connection_shutdown_total } } diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 34b2db3296c46..8726636cea55c 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -127,5 +127,7 @@ components: sources: statsd: { connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total invalid_record_total: components.sources.internal_metrics.output.metrics.invalid_record_total invalid_record_bytes_total: components.sources.internal_metrics.output.metrics.invalid_record_bytes_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total } } diff --git a/docs/reference/components/sources/stdin.cue b/docs/reference/components/sources/stdin.cue index 0415867208d0e..e3dace1be3bd7 100644 --- a/docs/reference/components/sources/stdin.cue +++ b/docs/reference/components/sources/stdin.cue @@ -102,6 +102,8 @@ components: sources: stdin: { } telemetry: metrics: { + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total stdin_reads_failed_total: components.sources.internal_metrics.output.metrics.stdin_reads_failed_total } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index 5c755ffbc38fe..e25a2a039bcb9 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -190,6 +190,8 @@ components: sources: syslog: { telemetry: metrics: { connection_read_errors_total: components.sources.internal_metrics.output.metrics.connection_read_errors_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total + processed_events_total: components.sources.internal_metrics.output.metrics.processed_events_total utf8_convert_errors_total: components.sources.internal_metrics.output.metrics.utf8_convert_errors_total } } diff --git a/docs/reference/components/transforms/add_fields.cue b/docs/reference/components/transforms/add_fields.cue index 7642033bd3474..076202e3c93d2 100644 --- a/docs/reference/components/transforms/add_fields.cue +++ b/docs/reference/components/transforms/add_fields.cue @@ -138,4 +138,8 @@ components: transforms: add_fields: { """ } } + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/coercer.cue b/docs/reference/components/transforms/coercer.cue index 02c4c43780d5d..8ebb65587966a 100644 --- a/docs/reference/components/transforms/coercer.cue +++ b/docs/reference/components/transforms/coercer.cue @@ -79,4 +79,8 @@ components: transforms: coercer: { } }, ] + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/concat.cue b/docs/reference/components/transforms/concat.cue index 294f7b9bec3b0..5dc9983be9527 100644 --- a/docs/reference/components/transforms/concat.cue +++ b/docs/reference/components/transforms/concat.cue @@ -87,4 +87,8 @@ components: transforms: concat: { } }, ] + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/filter.cue b/docs/reference/components/transforms/filter.cue index b9310d62ce45f..62dbe9b42cbb4 100644 --- a/docs/reference/components/transforms/filter.cue +++ b/docs/reference/components/transforms/filter.cue @@ -53,10 +53,6 @@ components: transforms: filter: { } } - telemetry: metrics: { - events_discarded_total: components.sources.internal_metrics.output.metrics.events_discarded_total - } - examples: [ { title: "Drop debug logs" @@ -84,4 +80,8 @@ components: transforms: filter: { ] }, ] + + telemetry: metrics: { + events_discarded_total: components.sources.internal_metrics.output.metrics.events_discarded_total + } } diff --git a/docs/reference/components/transforms/geoip.cue b/docs/reference/components/transforms/geoip.cue index 8393789f7e505..c4660bee34a36 100644 --- a/docs/reference/components/transforms/geoip.cue +++ b/docs/reference/components/transforms/geoip.cue @@ -261,4 +261,8 @@ components: transforms: geoip: { } } } + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/json_parser.cue b/docs/reference/components/transforms/json_parser.cue index 03920d5cf36ea..c323185846e34 100644 --- a/docs/reference/components/transforms/json_parser.cue +++ b/docs/reference/components/transforms/json_parser.cue @@ -146,4 +146,8 @@ components: transforms: json_parser: { ] } } + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/key_value_parser.cue b/docs/reference/components/transforms/key_value_parser.cue index f3718b69e2077..576c28dce2b41 100644 --- a/docs/reference/components/transforms/key_value_parser.cue +++ b/docs/reference/components/transforms/key_value_parser.cue @@ -187,4 +187,8 @@ components: transforms: key_value_parser: { } }, ] + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } diff --git a/docs/reference/components/transforms/log_to_metric.cue b/docs/reference/components/transforms/log_to_metric.cue index 98a3269cc6dbf..8184c9c0f7d77 100644 --- a/docs/reference/components/transforms/log_to_metric.cue +++ b/docs/reference/components/transforms/log_to_metric.cue @@ -412,4 +412,8 @@ components: transforms: log_to_metric: { """ } } + + telemetry: metrics: { + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total + } } From 30642958e7835604f2b18022a898cb3136c065c2 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Wed, 13 Jan 2021 09:53:55 -0600 Subject: [PATCH 002/769] fix(influxdb_metrics sink): Fix transmission of counter metrics (#5922) This reverts commit 95f767d02f92cb6e320b8dc51371dd02959f9499. After reverting this commit, the internal_metrics (and possibly other metrics sources) ends up sending its metrics through the metrics batch buffer twice before the batch is flushed to the sink. This allows the counter handling in the metrics batch buffer to generate the required incremental record in the batch, which then persists after the batch is reset. Without it, the metrics are sent once, no record is added to the batch, and it is never added later. Signed-off-by: Bruce Guenter --- src/sinks/util/service.rs | 34 ++++----- src/sinks/util/sink.rs | 156 +++++++++++++++++++++++++++++--------- 2 files changed, 137 insertions(+), 53 deletions(-) diff --git a/src/sinks/util/service.rs b/src/sinks/util/service.rs index 1d9570850b0ea..cc7b5f571cbd7 100644 --- a/src/sinks/util/service.rs +++ b/src/sinks/util/service.rs @@ -7,6 +7,7 @@ use super::{ Batch, BatchSink, Partition, PartitionBatchSink, }; use crate::buffers::Acker; +use futures::TryFutureExt; use serde::{ de::{self, Unexpected, Visitor}, Deserialize, Deserializer, Serialize, @@ -23,7 +24,7 @@ use tower::{ pub type Svc = RateLimit, AdaptiveConcurrencyLimit, L>>>; pub type TowerBatchedSink = BatchSink, B, Request>; -pub type TowerPartitionSink = PartitionBatchSink, B, K, Request>; +pub type TowerPartitionSink = PartitionBatchSink, K, Request>; pub trait ServiceBuilderExt { fn map(self, f: F) -> ServiceBuilder, L>> @@ -283,6 +284,10 @@ impl TowerRequestSettings { batch_timeout: Duration, acker: Acker, ) -> TowerBatchedSink + // Would like to return `impl Sink + SinkExt` here, but that + // doesn't work with later calls to `batched_with_min` etc (via + // `trait SinkExt` above), as it is missing a bound on the + // associated types that cannot be expressed in stable Rust. where L: RetryLogic, S: Service + Clone + Send + 'static, @@ -380,31 +385,31 @@ where pub struct Map { f: Arc R2 + Send + Sync + 'static>, - pub(crate) inner: S, + inner: S, } impl Service for Map where S: Service, + crate::Error: From, { type Response = S::Response; - type Error = S::Error; - type Future = S::Future; + type Error = crate::Error; + type Future = futures::future::MapErr crate::Error>; fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - self.inner.poll_ready(cx) + self.inner + .poll_ready(cx) + .map(|result| result.map_err(|e| e.into())) } fn call(&mut self, req: R1) -> Self::Future { let req = (self.f)(req); - self.inner.call(req) + self.inner.call(req).map_err(Into::into) } } -impl Clone for Map -where - S: Clone, -{ +impl Clone for Map { fn clone(&self) -> Self { Self { f: Arc::clone(&self.f), @@ -413,15 +418,6 @@ where } } -impl fmt::Debug for Map -where - S: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Map").field("inner", &self.inner).finish() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/sinks/util/sink.rs b/src/sinks/util/sink.rs index ee80bfd8ddd7a..048cb06a611f5 100644 --- a/src/sinks/util/sink.rs +++ b/src/sinks/util/sink.rs @@ -11,7 +11,7 @@ //! For each of these types this module provides one external type //! that can be used within sinks. The simplest type being the `StreamSink` //! type should be used when you do not want to batch events but you want -//! to _stream_ them to the downstream service. `BatchSink` and `PartitionBatchSink` +//! to _stream_ them to the downstream service. `BatchSink` and `PartitonBatchSink` //! are similar in the sense that they both take some `tower::Service`, `Batch` and //! `Acker` and will provide full batching, request dispatching and acking based on //! the settings passed. @@ -33,8 +33,7 @@ use super::{ batch::{Batch, PushResult, StatefulBatch}, - buffer::{Partition, PartitionBuffer, PartitionInnerBuffer}, - service::{Map, ServiceBuilderExt}, + buffer::partition::Partition, }; use crate::{buffers::Acker, Event}; use async_trait::async_trait; @@ -57,7 +56,7 @@ use tokio::{ sync::oneshot, time::{delay_for, Delay, Duration}, }; -use tower::{Service, ServiceBuilder}; +use tower::Service; use tracing_futures::Instrument; // === StreamSink === @@ -84,18 +83,16 @@ pub trait StreamSink { /// and r3 are dispatched and r2 and r3 complete, all events contained /// in all requests will not be acked until r1 has completed. #[pin_project] -#[derive(Debug)] pub struct BatchSink where B: Batch, { - #[pin] - inner: PartitionBatchSink< - Map, Request>, - PartitionBuffer, - (), - PartitionInnerBuffer, - >, + service: ServiceSink, + batch: StatefulBatch, + buffer: Option, + timeout: Duration, + linger: Option, + closing: bool, } impl BatchSink @@ -107,22 +104,25 @@ where B: Batch, { pub fn new(service: S, batch: B, timeout: Duration, acker: Acker) -> Self { - let service = ServiceBuilder::new() - .map(|req: PartitionInnerBuffer| req.into_parts().0) - .service(service); - let batch = PartitionBuffer::new(batch); - let inner = PartitionBatchSink::new(service, batch, timeout, acker); - Self { inner } + let service = ServiceSink::new(service, acker); + + Self { + service, + batch: batch.into(), + buffer: None, + timeout, + linger: None, + closing: false, + } } } -#[cfg(test)] impl BatchSink where B: Batch, { pub fn get_ref(&self) -> &S { - &self.inner.service.service.inner + &self.service.service } } @@ -136,21 +136,109 @@ where { type Error = crate::Error; - fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_ready(cx) + fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.buffer.is_some() { + match self.as_mut().poll_flush(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Pending => { + if self.buffer.is_some() { + return Poll::Pending; + } + } + } + } + + Poll::Ready(Ok(())) } - fn start_send(self: Pin<&mut Self>, item: B::Input) -> Result<(), Self::Error> { - let item = PartitionInnerBuffer::new(item, ()); - self.project().inner.start_send(item) + fn start_send(mut self: Pin<&mut Self>, item: B::Input) -> Result<(), Self::Error> { + if self.linger.is_none() { + trace!("Starting new batch timer."); + // We just inserted the first item of a new batch, so set our delay to the longest time + // we want to allow that item to linger in the batch before being flushed. + self.linger = Some(delay_for(self.timeout)); + } + + if let PushResult::Overflow(item) = self.batch.push(item) { + self.buffer = Some(item); + } + + Ok(()) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + // Poll inner service while not ready, if we don't have buffer or any batch. + if self.buffer.is_none() && self.batch.is_empty() { + ready!(self.service.poll_complete(cx)); + return Poll::Ready(Ok(())); + } + + // Try send batch. + if (self.closing && !self.batch.is_empty()) + || self.batch.was_full() + || self + .linger + .as_mut() + .map(|linger| matches!(linger.poll_unpin(cx), Poll::Ready(()))) + .unwrap_or(false) + { + let service_ready = match self.service.poll_ready(cx) { + Poll::Ready(Ok(())) => true, + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Pending => false, + }; + if service_ready { + trace!("Service ready; Sending batch."); + + let batch = self.batch.fresh_replace(); + self.linger = None; + + let batch_size = batch.num_items(); + let request = batch.finish(); + tokio::spawn(self.service.call(request, batch_size)); + + continue; + } + } + + // Try move buffer to batch. + if self.batch.is_empty() { + if let Some(item) = self.buffer.take() { + self.as_mut().start_send(item)?; + if self.buffer.is_some() { + unreachable!("Empty buffer overflowed."); + } + + continue; + } + } + + // Only poll inner service and return `Poll::Pending` anyway. + ready!(self.service.poll_complete(cx)); + return Poll::Pending; + } } - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_flush(cx) + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + trace!("Closing batch sink."); + self.closing = true; + self.poll_flush(cx) } +} - fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_close(cx) +impl fmt::Debug for BatchSink +where + S: fmt::Debug, + B: Batch + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BatchSink") + .field("service", &self.service) + .field("batch", &self.batch) + .field("timeout", &self.timeout) + .finish() } } @@ -171,7 +259,7 @@ where /// and r3 are dispatched and r2 and r3 complete, all events contained /// in all requests will not be acked until r1 has completed. #[pin_project] -pub struct PartitionBatchSink +pub struct PartitionBatchSink where B: Batch, { @@ -184,7 +272,7 @@ where closing: bool, } -impl PartitionBatchSink +impl PartitionBatchSink where B: Batch, B::Input: Partition, @@ -209,7 +297,7 @@ where } } -impl Sink for PartitionBatchSink +impl Sink for PartitionBatchSink where B: Batch, B::Input: Partition, @@ -338,13 +426,13 @@ where } } -impl fmt::Debug for PartitionBatchSink +impl fmt::Debug for PartitionBatchSink where S: fmt::Debug, B: Batch + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PartitionBatchSink") + f.debug_struct("PartitionedBatchSink") .field("service", &self.service) .field("batch", &self.batch) .field("timeout", &self.timeout) From 28155e37848feab7ab59fb8fe7a0818c5dce78b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thor=20Anker=20Kvisg=C3=A5rd=20Lange?= Date: Wed, 13 Jan 2021 17:30:29 +0100 Subject: [PATCH 003/769] fix(kubernetes platform): Adjusting pod security policy to allow vector-agent to read host logs (#6019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thor Anker Kvisgård Lange --- .../helm/vector-agent/templates/podsecuritypolicy.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/distribution/helm/vector-agent/templates/podsecuritypolicy.yaml b/distribution/helm/vector-agent/templates/podsecuritypolicy.yaml index 87f6912dd4763..010aa6c26966b 100644 --- a/distribution/helm/vector-agent/templates/podsecuritypolicy.yaml +++ b/distribution/helm/vector-agent/templates/podsecuritypolicy.yaml @@ -13,12 +13,14 @@ spec: - 'hostPath' - 'configMap' - 'secret' + - 'projected' allowedHostPaths: - pathPrefix: "/var/log" readOnly: true - - pathPrefix: "/var/lib/docker/containers" + - pathPrefix: "/var/lib" readOnly: true - - pathPrefix: "/var/lib/vector/" + - pathPrefix: "/var/lib/vector" + readOnly: false {{- range .Values.extraVolumes }} {{- if .hostPath }} - pathPrefix: {{ .hostPath.path }} @@ -28,7 +30,7 @@ spec: hostIPC: false hostPID: false runAsUser: - rule: 'MustRunAsNonRoot' + rule: 'RunAsAny' seLinux: rule: 'RunAsAny' supplementalGroups: From e784d59040153f84cee43df429832f55c36f8681 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 13 Jan 2021 20:40:14 +0300 Subject: [PATCH 004/769] fix(ci): correct start/stop instructions (#6029) Signed-off-by: Kirill Fomichev --- scripts/setup_integration/humio_integration_env.sh | 1 - scripts/setup_integration/mongodb_metrics_integration_env.sh | 4 ++-- scripts/setup_integration/nginx_integration_env.sh | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/setup_integration/humio_integration_env.sh b/scripts/setup_integration/humio_integration_env.sh index d4a16fb3170cd..394f574e65d12 100755 --- a/scripts/setup_integration/humio_integration_env.sh +++ b/scripts/setup_integration/humio_integration_env.sh @@ -35,7 +35,6 @@ stop_podman () { } stop_docker () { - docker rm --force vector_humio 2>/dev/null; true docker rm --force vector_humio 2>/dev/null; true docker network rm vector-test-integration-humio 2>/dev/null; true } diff --git a/scripts/setup_integration/mongodb_metrics_integration_env.sh b/scripts/setup_integration/mongodb_metrics_integration_env.sh index 94f754432bfbb..552d085845459 100755 --- a/scripts/setup_integration/mongodb_metrics_integration_env.sh +++ b/scripts/setup_integration/mongodb_metrics_integration_env.sh @@ -39,13 +39,13 @@ start_docker () { } stop_podman () { - podman rm --force vector_mongodb_metrics1 2>/dev/null; true + podman rm --force vector_mongodb_metrics1 vector_mongodb_metrics2 2>/dev/null; true podman pod stop vector-test-integration-mongodb_metrics 2>/dev/null; true podman pod rm --force vector-test-integration-mongodb_metrics 2>/dev/null; true } stop_docker () { - docker rm --force vector_mongodb_metrics1 2>/dev/null; true + docker rm --force vector_mongodb_metrics1 vector_mongodb_metrics2 2>/dev/null; true docker network rm vector-test-integration-mongodb_metrics 2>/dev/null; true } diff --git a/scripts/setup_integration/nginx_integration_env.sh b/scripts/setup_integration/nginx_integration_env.sh index 1bdfab048a2f4..05fa3e63f6782 100755 --- a/scripts/setup_integration/nginx_integration_env.sh +++ b/scripts/setup_integration/nginx_integration_env.sh @@ -31,12 +31,13 @@ start_docker () { } stop_podman () { + podman rm --force vector_nginx 2>/dev/null; true podman pod stop vector-test-integration-nginx 2>/dev/null; true podman pod rm --force vector-test-integration-nginx 2>/dev/null; true } stop_docker () { - docker rm --force vector_nats 2>/dev/null; true + docker rm --force vector_nginx 2>/dev/null; true docker network rm vector-test-integration-nginx 2>/dev/null; true } From 86263f5fcf3aad1fa77a954cc9676379a7cda443 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 13 Jan 2021 14:40:49 -0400 Subject: [PATCH 005/769] chore(ci): Ensure target/criterion exists (#6030) It does not exist if pulling the master artifacts fails. Signed-off-by: Jesse Szwedko --- .github/workflows/benches.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 73ad3365b34ed..dad0114faaa8e 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -76,7 +76,9 @@ jobs: # run benchmarks on isolated CPU and with address randomization # disabled - name: Run benchmarks - run: setarch $(uname -m) -R taskset -c "$(cat /sys/devices/system/cpu/isolated)" make bench-all | tee target/criterion/out + run: | + mkdir -p target/criterion + setarch $(uname -m) -R taskset -c "$(cat /sys/devices/system/cpu/isolated)" make bench-all | tee target/criterion/out - run: zip --recurse-paths target/criterion.zip target/criterion - uses: actions/upload-artifact@v2 with: From 3a9f7e6ceebff6c06f95d17bf93b3636df0f8351 Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Wed, 13 Jan 2021 22:55:27 +0300 Subject: [PATCH 006/769] chore(deps): update (#5999) Signed-off-by: Kirill Fomichev --- Cargo.lock | 457 ++++++++++---------- Cargo.toml | 2 +- lib/vector-api-client/src/gql/components.rs | 16 +- src/api/schema/metrics/source/file.rs | 2 +- 4 files changed, 237 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba1c0257cda52..63e211446f55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,9 +81,9 @@ checksum = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" [[package]] name = "anyhow" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee67c11feeac938fae061b232e38e0b6d94f97a9df10e6271319325ac4c56a86" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" [[package]] name = "approx" @@ -160,7 +160,7 @@ dependencies = [ "flate2", "futures-core", "memchr", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "tokio", "zstd", "zstd-safe", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "2.4.4" +version = "2.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a790df528e46754fd6e2cac07bd185b90382c6fb2f83d4b53636da46f5350e82" +checksum = "46a20530b1e9f3a4a33ba6a78384a18d1505a91515cf1d5aa2d94b463ffb851f" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -210,31 +210,32 @@ dependencies = [ "futures-channel", "futures-timer", "futures-util", + "http", "indexmap", "log", "lru", "multer", "num-traits", "once_cell", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "regex", "serde", "serde_json", "sha2 0.9.2", - "spin 0.7.0", + "spin 0.7.1", "static_assertions", "tempfile", "thiserror", "tracing 0.1.22", "url", - "uuid 0.8.1", + "uuid 0.8.2", ] [[package]] name = "async-graphql-derive" -version = "2.4.3" +version = "2.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b938025ca3bda492560d9dd88f9b2e2ab1d28b86ece7c7faf0fd1e0940d6720e" +checksum = "a75763cc369e523724e703a6077d9a913f4bd74741ec8d0f08ad1a3c6a11cd91" dependencies = [ "Inflector", "async-graphql-parser", @@ -242,15 +243,15 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "thiserror", ] [[package]] name = "async-graphql-parser" -version = "2.1.2" +version = "2.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee04e49c932b12a7a18163a59ee5596a83422e4b91cb64ca63e3545bd4c4560e" +checksum = "7a33dd3bc63cff9e57ab8c1fb3d6a396d277fb051937ec0298281d8ad0ea39dd" dependencies = [ "async-graphql-value", "pest", @@ -271,9 +272,9 @@ dependencies = [ [[package]] name = "async-graphql-warp" -version = "2.4.4" +version = "2.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd150f2030d69f6341feefd1fad320227381065b041eac910934c84e6ebb7b7" +checksum = "ea142666cbee77a4b972ecc1cedc0eee47f57aa8a7397bbf0641f0227c3ce6a3" dependencies = [ "async-graphql", "futures-util", @@ -367,7 +368,7 @@ checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -384,7 +385,7 @@ checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -442,7 +443,7 @@ dependencies = [ "strum_macros", "thiserror", "typed-builder 0.5.1", - "uuid 0.8.1", + "uuid 0.8.2", "zerocopy", ] @@ -738,9 +739,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" @@ -761,6 +762,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" + [[package]] name = "bytesize" version = "1.0.1" @@ -987,9 +994,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" [[package]] name = "constant_time_eq" @@ -1439,19 +1446,19 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c88d9506e2e9230f6107701b7d8425f4cb3f6df108ec3042a26e936666da5" +checksum = "10bcb9d7dcbf7002aaffbb53eac22906b64cdcc127971dcc387d8eb7c95d5560" dependencies = [ "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] name = "curve25519-dalek" -version = "3.0.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8492de420e9e60bc9a1d66e2dbb91825390b738a388606600663fc529b4b307" +checksum = "f627126b946c25a4638eec0ea634fc52506dea98db118aae985118ce7c3d723f" dependencies = [ "byteorder", "digest 0.9.0", @@ -1490,7 +1497,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", "strsim 0.9.3", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1501,7 +1508,7 @@ checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ "darling_core", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1537,9 +1544,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b937cd1fbd1f194ac842196bd2529f21618088ee6d8bff6a46ece611451c96b" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" dependencies = [ "cfg-if 1.0.0", "num_cpus", @@ -1582,7 +1589,7 @@ checksum = "eaed5874effa6cde088c644ddcdcb4ffd1511391c5be4fdd7a5ccd02c7e4a183" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1592,7 +1599,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1603,7 +1610,7 @@ checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1667,18 +1674,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" dependencies = [ "libc", - "redox_users", + "redox_users 0.3.5", "winapi 0.3.9", ] [[package]] name = "dirs-sys-next" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99de365f605554ae33f115102a02057d4fc18b01f3284d6870be0938743cfe7d" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.0", "winapi 0.3.9", ] @@ -1765,7 +1772,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -1810,7 +1817,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", "rustversion", - "syn 1.0.57", + "syn 1.0.58", "synstructure", ] @@ -1865,7 +1872,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "synstructure", ] @@ -1902,9 +1909,9 @@ dependencies = [ "bytes 0.5.6", "chrono", "crc", - "dashmap 4.0.1", + "dashmap 4.0.2", "flate2", - "futures 0.3.8", + "futures 0.3.9", "glob 0.3.0", "indexmap", "libc", @@ -1927,7 +1934,7 @@ checksum = "0c122a393ea57648015bf06fbd3d372378992e86b9ff5a7a497b076a28c79efe" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "winapi 0.3.9", ] @@ -2051,9 +2058,9 @@ checksum = "4c7e4c2612746b0df8fed4ce0c69156021b704c9aefa360311c04e6e9e002eed" [[package]] name = "futures" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "c70be434c505aee38639abccb918163b63158a4b4bb791b45b7023044bdc3c9c" dependencies = [ "futures-channel", "futures-core", @@ -2066,9 +2073,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "f01c61843314e95f96cc9245702248733a3a3d744e43e2e755e3c7af8348a0a9" dependencies = [ "futures-core", "futures-sink", @@ -2076,15 +2083,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "db8d3b0917ff63a2a96173133c02818fac4a746b0a57569d3baca9ec0e945e08" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "9ee9ca2f7eb4475772cf39dd1cd06208dce2670ad38f4d9c7262b3e15f127068" dependencies = [ "futures-core", "futures-task", @@ -2104,9 +2111,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "e37c1a51b037b80922864b8eed90692c5cd8abd4c71ce49b77146caa47f3253b" [[package]] name = "futures-lite" @@ -2119,33 +2126,33 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "waker-fn", ] [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "0f8719ca0e1f3c5e34f3efe4570ef2c0610ca6da85ae7990d472e9cbfba13664" dependencies = [ "proc-macro-hack", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "f6adabac1290109cfa089f79192fb6244ad2c3f1cc2281f3e1dd987592b71feb" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +checksum = "a92a0843a2ff66823a8f7c77bffe9a09be2b64e533562c412d63075643ec0038" dependencies = [ "once_cell", ] @@ -2158,9 +2165,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "036a2107cdeb57f6d7322f1b6c363dad67cd63ca3b7d1b925bdf75bd5d96cda9" dependencies = [ "futures 0.1.30", "futures-channel", @@ -2170,7 +2177,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite 0.2.4", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -2178,19 +2185,6 @@ dependencies = [ "tokio-io", ] -[[package]] -name = "generator" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" -dependencies = [ - "cc", - "libc", - "log", - "rustc_version", - "winapi 0.3.9", -] - [[package]] name = "generic-array" version = "0.12.3" @@ -2241,7 +2235,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -2252,7 +2246,7 @@ checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -2302,7 +2296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c55b7ac37895bd6e4ca0b357c074248358c95e20cf1cf2b462603121f7b87" dependencies = [ "arc-swap", - "futures 0.3.8", + "futures 0.3.9", "log", "reqwest", "serde", @@ -2310,7 +2304,7 @@ dependencies = [ "serde_json", "simpl", "smpl_jwt", - "time 0.2.23", + "time 0.2.24", "tokio", ] @@ -2360,7 +2354,7 @@ dependencies = [ "quote 1.0.8", "serde", "serde_json", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -2372,7 +2366,7 @@ dependencies = [ "failure", "graphql_client_codegen", "proc-macro2 1.0.24", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -2504,7 +2498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba5fb13a3b90581d22b4edf99e87c54316444622ae123d36816a227a7caa6df" dependencies = [ "cfg-if 1.0.0", - "futures 0.3.8", + "futures 0.3.9", "glob 0.3.0", "heim-common", "heim-runtime", @@ -2590,7 +2584,7 @@ dependencies = [ "async-trait", "cfg-if 1.0.0", "darwin-libproc", - "futures 0.3.8", + "futures 0.3.9", "heim-common", "heim-cpu", "heim-host", @@ -2612,7 +2606,7 @@ version = "0.1.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54ec7e5238c8f0dd0cc60914d31a5a7aadd4cde74c966a76c1caed1f5224e9b8" dependencies = [ - "futures 0.3.8", + "futures 0.3.9", "futures-timer", "once_cell", "smol", @@ -2636,7 +2630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7616c8db704e85495952fc1350da7624ebf1d459feab1b7f01d43944b7412744" dependencies = [ "cfg-if 1.0.0", - "futures 0.3.8", + "futures 0.3.9", "heim-common", "heim-runtime", "raw-cpuid 8.1.2", @@ -2700,11 +2694,11 @@ dependencies = [ [[package]] name = "http" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84129d298a6d57d246960ff8eb831ca4af3f96d29e2e28848dae275408658e26" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 0.5.6", + "bytes 1.0.1", "fnv", "itoa", ] @@ -2761,7 +2755,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", + "pin-project 1.0.4", "socket2", "tokio", "tower-service", @@ -2927,7 +2921,7 @@ checksum = "75c094e94816723ab936484666968f5b58060492e880f3c8d00489a1e244fa51" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -3039,7 +3033,7 @@ checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" name = "k8s-e2e-tests" version = "0.1.0" dependencies = [ - "futures 0.3.8", + "futures 0.3.9", "k8s-openapi", "k8s-test-framework", "regex", @@ -3157,9 +3151,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" [[package]] name = "libflate" @@ -3231,9 +3225,9 @@ dependencies = [ [[package]] name = "linked-hash-map" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" [[package]] name = "linked_hash_set" @@ -3275,9 +3269,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" dependencies = [ "cfg-if 0.1.10", ] @@ -3288,19 +3282,6 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879777f0cc6f3646a044de60e4ab98c75617e3f9580f7a2032e6ad7ea0cd3054" -[[package]] -name = "loom" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" -dependencies = [ - "cfg-if 0.1.10", - "generator", - "scoped-tls", - "serde", - "serde_json", -] - [[package]] name = "lru" version = "0.6.3" @@ -3385,7 +3366,7 @@ version = "0.7.0-dev" source = "git+https://github.com/bytecodealliance/lucet.git?rev=b1863dacc8c92c11e5434fc8815d9b9a26cfe3db#b1863dacc8c92c11e5434fc8815d9b9a26cfe3db" dependencies = [ "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -3416,7 +3397,7 @@ dependencies = [ "lucet-wiggle", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wasi-common", "wiggle-generate", ] @@ -3441,7 +3422,7 @@ dependencies = [ "lucet-module", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wiggle-generate", "witx", ] @@ -3453,7 +3434,7 @@ source = "git+https://github.com/bytecodealliance/lucet.git?rev=b1863dacc8c92c11 dependencies = [ "lucet-wiggle-generate", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wiggle-generate", "witx", ] @@ -3625,7 +3606,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", "regex", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -3799,7 +3780,7 @@ dependencies = [ "chrono", "derivative 2.1.3", "err-derive", - "futures 0.3.8", + "futures 0.3.9", "futures-intrusive", "hex", "hmac 0.7.1", @@ -3826,7 +3807,7 @@ dependencies = [ "trust-dns-proto", "trust-dns-resolver", "typed-builder 0.3.0", - "uuid 0.8.1", + "uuid 0.8.2", "version_check 0.9.2", "webpki", "webpki-roots 0.18.0", @@ -3841,7 +3822,7 @@ dependencies = [ "bytes 0.5.6", "derive_more", "encoding_rs", - "futures 0.3.8", + "futures 0.3.9", "http", "httparse", "lazy_static", @@ -4067,7 +4048,7 @@ checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -4140,7 +4121,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -4330,7 +4311,7 @@ dependencies = [ "cfg-if 0.1.10", "cloudabi", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi 0.3.9", ] @@ -4344,7 +4325,7 @@ dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi 0.3.9", ] @@ -4435,7 +4416,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -4470,11 +4451,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal 1.0.4", ] [[package]] @@ -4485,18 +4466,18 @@ checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] name = "pin-project-internal" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -4507,9 +4488,9 @@ checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" [[package]] name = "pin-project-lite" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" [[package]] name = "pin-utils" @@ -4558,7 +4539,7 @@ dependencies = [ name = "portpicker" version = "1.0.0" dependencies = [ - "rand 0.8.0", + "rand 0.8.1", ] [[package]] @@ -4623,7 +4604,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "version_check 0.9.2", ] @@ -4714,7 +4695,7 @@ dependencies = [ "itertools 0.8.2", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -4737,7 +4718,7 @@ dependencies = [ "bytes 0.5.6", "chrono", "crc", - "futures 0.3.8", + "futures 0.3.9", "futures-io", "futures-timer", "log", @@ -4846,13 +4827,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76330fb486679b4ace3670f117bbc9e16204005c4bde9c4bd372f45bed34f12" +checksum = "c24fcd450d3fa2b592732565aa4f17a27a61c65ece4726353e000939b0edee34" dependencies = [ "libc", "rand_chacha 0.3.0", - "rand_core 0.6.0", + "rand_core 0.6.1", "rand_hc 0.3.0", ] @@ -4883,7 +4864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.0", + "rand_core 0.6.1", ] [[package]] @@ -4912,9 +4893,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8b34ba8cfb21243bd8df91854c830ff0d785fff2e82ebd4434c2644cb9ada18" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" dependencies = [ "getrandom 0.2.1", ] @@ -4926,7 +4907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9e8f32ad24fb80d07d2323a9a2ce8b30d68a62b8cb4df88119ff49a698f038" dependencies = [ "num-traits", - "rand 0.8.0", + "rand 0.8.1", ] [[package]] @@ -4953,7 +4934,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core 0.6.0", + "rand_core 0.6.1", ] [[package]] @@ -5073,7 +5054,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db594dc221933be6f2ad804b997b48a57a63436c26ab924222c28e9a36ad210a" dependencies = [ - "futures 0.3.8", + "futures 0.3.9", "libc", "log", "rdkafka-sys", @@ -5114,6 +5095,15 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_syscall" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.3.5" @@ -5121,10 +5111,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom 0.1.16", - "redox_syscall", + "redox_syscall 0.1.57", "rust-argon2", ] +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom 0.2.1", + "redox_syscall 0.2.4", +] + [[package]] name = "regalloc" version = "0.0.26" @@ -5138,9 +5138,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -5160,9 +5160,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remap-cli" @@ -5204,7 +5204,7 @@ dependencies = [ "syslog_loose", "tracing 0.1.22", "url", - "uuid 0.8.1", + "uuid 0.8.2", ] [[package]] @@ -5257,7 +5257,7 @@ dependencies = [ "mime_guess", "native-tls", "percent-encoding", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "rustls 0.18.1", "serde", "serde_json", @@ -5334,7 +5334,7 @@ checksum = "c5057642d3b77125bdae53ac923ed46c931a4387a9409d192dedd6ae03926dc6" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde_urlencoded 0.6.1", "xml-rs", @@ -5351,7 +5351,7 @@ dependencies = [ "bytes 0.5.6", "crc32fast", "flate2", - "futures 0.3.8", + "futures 0.3.9", "http", "hyper", "hyper-tls", @@ -5378,7 +5378,7 @@ dependencies = [ "async-trait", "chrono", "dirs", - "futures 0.3.8", + "futures 0.3.9", "hyper", "pin-project 0.4.27", "regex", @@ -5397,7 +5397,7 @@ checksum = "12a49c8259ada24cb2c64d2a2aee0ca150eed3aef91d102063d8efca747ccaa3" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde", "serde_derive", @@ -5412,7 +5412,7 @@ checksum = "8e3c0c212bda09bad1e6b34cf2cdc9b6960214ff52935300567984c80389e29a" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde", "serde_json", @@ -5426,7 +5426,7 @@ checksum = "eb8a3baabc67851ca472d25c9fe848b229969a1412cbc50ad00ce3af00e9aeae" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde", "serde_json", @@ -5440,7 +5440,7 @@ checksum = "47079cf81cf7301d96195e495f2649cd731bedfe592925741997eb5a02d253e6" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde", "serde_json", @@ -5454,7 +5454,7 @@ checksum = "1146e37a7c1df56471ea67825fe09bbbd37984b5f6e201d8b2e0be4ee15643d8" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "xml-rs", ] @@ -5467,7 +5467,7 @@ checksum = "97a740a88dde8ded81b6f2cff9cd5e054a5a2e38a38397260f7acdd2c85d17dd" dependencies = [ "base64 0.12.3", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "hex", "hmac 0.8.1", "http", @@ -5480,7 +5480,7 @@ dependencies = [ "rustc_version", "serde", "sha2 0.9.2", - "time 0.2.23", + "time 0.2.24", "tokio", ] @@ -5492,7 +5492,7 @@ checksum = "fcd228a1b4ce3f3a40541ee8cef526ff3702b58a4779fb05c31e739174efda5e" dependencies = [ "async-trait", "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde_urlencoded 0.6.1", "xml-rs", @@ -5507,7 +5507,7 @@ dependencies = [ "async-trait", "bytes 0.5.6", "chrono", - "futures 0.3.8", + "futures 0.3.9", "rusoto_core", "serde_urlencoded 0.6.1", "tempfile", @@ -5791,7 +5791,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ - "semver-parser 0.10.1", + "semver-parser 0.10.2", "serde", ] @@ -5803,18 +5803,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "semver-parser" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ef146c2ad5e5f4b037cd6ce2ebb775401729b19a82040c1beac9d36c7d1428" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" dependencies = [ "pest", ] [[package]] name = "serde" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "9bdd36f49e35b61d49efd8aa7fc068fd295961fd2286d0b2ee9a4c7a14e99cc3" dependencies = [ "serde_derive", ] @@ -5860,13 +5860,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "552954ce79a059ddd5fd68c271592374bd15cab2274970380c000118aeffe1cd" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -5924,14 +5924,14 @@ dependencies = [ "darling", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] name = "serde_yaml" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7baae0a99f1a324984bcdc5f0718384c1f69775f1c7eec8b859b71b443e3fd7" +checksum = "971be8f6e4d4a47163b405a3df70d14359186f9ab0f3a3ec37df144ca1ce089f" dependencies = [ "dtoa", "linked-hash-map", @@ -6009,12 +6009,11 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" +checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" dependencies = [ "lazy_static", - "loom", ] [[package]] @@ -6079,9 +6078,9 @@ dependencies = [ [[package]] name = "signature" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f060a7d147e33490ec10da418795238fd7545bba241504d6b31a409f2e6210" +checksum = "0f0242b8e50dd9accdd56170e94ca1ebd223b098eb9c83539a6e367d0f36ae68" [[package]] name = "simpl" @@ -6138,7 +6137,7 @@ dependencies = [ "serde_derive", "serde_json", "simpl", - "time 0.2.23", + "time 0.2.24", ] [[package]] @@ -6162,7 +6161,7 @@ checksum = "1508efa03c362e23817f96cde18abed596a25219a8b2c66e8db33c03543d315b" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6190,9 +6189,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652ac3743312871a5fb703f0337e68ffa3cdc28c863efad0b8dc858fa10c991b" +checksum = "13287b4da9d1207a4f4929ac390916d64eacfe236a487e9a9f5b3be392be5162" [[package]] name = "standback" @@ -6233,7 +6232,7 @@ dependencies = [ "quote 1.0.8", "serde", "serde_derive", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6249,7 +6248,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6328,7 +6327,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6346,7 +6345,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6383,9 +6382,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.57" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4211ce9909eb971f111059df92c45640aad50a619cf55cd76476be803c4c68e6" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", @@ -6400,7 +6399,7 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "unicode-xid 0.2.1", ] @@ -6446,14 +6445,14 @@ checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand 0.7.3", - "redox_syscall", + "rand 0.8.1", + "redox_syscall 0.2.4", "remove_dir_all", "winapi 0.3.9", ] @@ -6493,7 +6492,7 @@ checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6518,9 +6517,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b" +checksum = "273d3ed44dca264b0d6b3665e8d48fb515042d42466fad93d2a45b90ec4058f7" dependencies = [ "const_fn", "libc", @@ -6551,7 +6550,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", "standback", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6641,7 +6640,7 @@ checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6837,7 +6836,7 @@ checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite 0.2.4", "tracing-attributes 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "tracing-core 0.1.17", ] @@ -6850,7 +6849,7 @@ checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6860,7 +6859,7 @@ source = "git+https://github.com/tokio-rs/tracing?rev=f470db1b0354b368f62f9ee4d7 dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -6887,7 +6886,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" dependencies = [ "futures 0.1.30", - "futures 0.3.8", + "futures 0.3.9", "futures-task", "pin-project 0.4.27", "tracing 0.1.22", @@ -6961,7 +6960,7 @@ name = "tracing-tower" version = "0.1.0" source = "git+https://github.com/tokio-rs/tracing?rev=f470db1b0354b368f62f9ee4d763595d16373231#f470db1b0354b368f62f9ee4d763595d16373231" dependencies = [ - "futures 0.3.8", + "futures 0.3.9", "http", "pin-project 0.4.27", "tower-layer", @@ -6986,7 +6985,7 @@ dependencies = [ "async-trait", "backtrace", "enum-as-inner", - "futures 0.3.8", + "futures 0.3.9", "idna", "lazy_static", "log", @@ -7005,7 +7004,7 @@ checksum = "6759e8efc40465547b0dfce9500d733c65f969a4cbbfbe3ccf68daaa46ef179e" dependencies = [ "backtrace", "cfg-if 0.1.10", - "futures 0.3.8", + "futures 0.3.9", "ipconfig", "lazy_static", "log", @@ -7085,7 +7084,7 @@ checksum = "78cea224ddd4282dfc40d1edabbd0c020a12e946e3a48e2c2b8f6ff167ad29fe" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -7115,7 +7114,7 @@ checksum = "3f2466fc87b07b800a5060f89ba579d6882f7a03ac21363e4737764aaf9f99f9" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", ] [[package]] @@ -7284,11 +7283,11 @@ dependencies = [ [[package]] name = "uuid" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "rand 0.7.3", + "getrandom 0.2.1", "serde", ] @@ -7347,7 +7346,7 @@ dependencies = [ "file-source", "flate2", "futures 0.1.30", - "futures 0.3.8", + "futures 0.3.9", "getset", "glob 0.3.0", "goauth", @@ -7395,7 +7394,7 @@ dependencies = [ "percent-encoding", "pest", "pest_derive", - "pin-project 1.0.2", + "pin-project 1.0.4", "portpicker", "pretty_assertions", "prometheus-parser", @@ -7403,7 +7402,7 @@ dependencies = [ "prost-build", "prost-types", "pulsar", - "rand 0.8.0", + "rand 0.8.1", "rand_distr", "rdkafka", "regex", @@ -7463,7 +7462,7 @@ dependencies = [ "tui", "typetag", "url", - "uuid 0.8.1", + "uuid 0.8.2", "vector-api-client", "vector-wasm", "walkdir", @@ -7478,7 +7477,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "futures 0.3.8", + "futures 0.3.9", "graphql_client", "reqwest", "serde", @@ -7486,7 +7485,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "url", - "uuid 0.8.1", + "uuid 0.8.2", "weak-table", ] @@ -7593,7 +7592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" dependencies = [ "bytes 0.5.6", - "futures 0.3.8", + "futures 0.3.9", "headers", "http", "hyper", @@ -7669,7 +7668,7 @@ dependencies = [ "log", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wasm-bindgen-shared", ] @@ -7703,7 +7702,7 @@ checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7834,7 +7833,7 @@ dependencies = [ "heck", "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "witx", ] @@ -7844,7 +7843,7 @@ version = "0.18.0" source = "git+https://github.com/bytecodealliance/lucet.git?rev=b1863dacc8c92c11e5434fc8815d9b9a26cfe3db#b1863dacc8c92c11e5434fc8815d9b9a26cfe3db" dependencies = [ "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "wiggle-generate", "witx", ] @@ -8003,7 +8002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ "proc-macro2 1.0.24", - "syn 1.0.57", + "syn 1.0.58", "synstructure", ] @@ -8024,7 +8023,7 @@ checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.8", - "syn 1.0.57", + "syn 1.0.58", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index ab4ebdda78d27..232213f44b748 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -585,7 +585,7 @@ harness = false required-features = ["remap-benches"] [patch.'https://github.com/tower-rs/tower'] -tower-layer = "0.3" +tower-layer = "=0.3.0" [patch.crates-io] # TODO: update to next 0.12.x (after 0.12.0, if any) diff --git a/lib/vector-api-client/src/gql/components.rs b/lib/vector-api-client/src/gql/components.rs index e3fe1db143b8d..f8696e308152a 100644 --- a/lib/vector-api-client/src/gql/components.rs +++ b/lib/vector-api-client/src/gql/components.rs @@ -1,7 +1,7 @@ use crate::{BoxedSubscription, QueryResult}; use async_trait::async_trait; use graphql_client::GraphQLQuery; -use serde::export::Formatter; +use std::fmt; /// Components query for returning sources, transforms, and sinks #[derive(GraphQLQuery, Debug, Copy, Clone)] @@ -115,8 +115,8 @@ impl components_query::ComponentsQueryComponentsEdgesNodeOn { } } -impl std::fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let res = match self { components_query::ComponentsQueryComponentsEdgesNodeOn::Source(_) => "source", components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => "transform", @@ -127,10 +127,8 @@ impl std::fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeO } } -impl std::fmt::Display - for component_added_subscription::ComponentAddedSubscriptionComponentAddedOn -{ - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for component_added_subscription::ComponentAddedSubscriptionComponentAddedOn { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let res = match self { component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Source => { "source" @@ -147,10 +145,10 @@ impl std::fmt::Display } } -impl std::fmt::Display +impl fmt::Display for component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let res = match self { component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Source => { "source" diff --git a/src/api/schema/metrics/source/file.rs b/src/api/schema/metrics/source/file.rs index fc33dad1de564..f18e3216deca4 100644 --- a/src/api/schema/metrics/source/file.rs +++ b/src/api/schema/metrics/source/file.rs @@ -21,7 +21,7 @@ impl<'a> FileSourceMetricFile<'a> { } #[Object] -impl FileSourceMetricFile<'_> { +impl<'a> FileSourceMetricFile<'a> { /// File name async fn name(&self) -> &str { &*self.name From 910eceb5e715e6c4dd60b12fee539b67d6490f95 Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Thu, 14 Jan 2021 08:06:02 +0100 Subject: [PATCH 007/769] chore(remap): add error-handling behavior tests (#5958) Signed-off-by: Jean Mertz --- tests/behavior/transforms/remap.toml | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index fe2337f51b2c5..7635415594513 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -201,6 +201,63 @@ [[tests.outputs.conditions]] "a.b\\.c.equals" = "baz" +[transforms.remap_infallible_assignment] + inputs = [] + type = "remap" + source = """ + .nope, .err1 = parse_json("{ INVALID }") + .ok, .err2 = parse_json("{ \\"foo\\": true }") + """ +[[tests]] + name = "remap_infallible_assignment" + [tests.input] + insert_at = "remap_infallible_assignment" + type = "raw" + value = "" + [[tests.outputs]] + extract_from = "remap_infallible_assignment" + [[tests.outputs.conditions]] + "nope.equals" = "" + "err1.equals" = "function call error: unable to parse json: key must be a string at line 1 column 3" + "ok.foo.equals" = true + "err2.equals" = "" + +[transforms.remap_error_coalesce_operator] + inputs = [] + type = "remap" + source = """ + .val1 = parse_json("{ INVALID }") ?? "nope" + .val2 = parse_json("true") ?? "nope" + """ +[[tests]] + name = "remap_error_coalesce_operator" + [tests.input] + insert_at = "remap_error_coalesce_operator" + type = "raw" + value = "" + [[tests.outputs]] + extract_from = "remap_error_coalesce_operator" + [[tests.outputs.conditions]] + "val1.equals" = "nope" + "val2.equals" = true + +[transforms.remap_bang_function] + inputs = [] + type = "remap" + source = """ + .val = parse_json!("{ INVALID }") + """ +[[tests]] + name = "remap_bang_function" + [tests.input] + insert_at = "remap_bang_function" + type = "raw" + value = "" + [[tests.outputs]] + extract_from = "remap_bang_function" + [[tests.outputs.conditions]] + "val.exists" = false + [transforms.remap_function_arguments] inputs = [] type = "remap" From c5c7faef37afe68c508c02871510f932078f65e9 Mon Sep 17 00:00:00 2001 From: Kruno Tomola Fabro Date: Thu, 14 Jan 2021 08:07:52 +0100 Subject: [PATCH 008/769] chore(journald source): Deprecate `remap_priority` option (#5964) Signed-off-by: ktf --- docs/reference/components/sources/journald.cue | 7 ------- src/sources/journald.rs | 7 ++++++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index 460282d765ac8..9db34652b78ed 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -93,13 +93,6 @@ components: sources: journald: { examples: ["/usr/local/bin/journalctl"] } } - remap_priority: { - common: false - description: "If the record from journald contains a `PRIORITY` field, it will be remapped into the equivalent syslog priority level name using the standard (abbreviated) all-capitals names such as `EMERG` or `ERR`." - required: false - warnings: [] - type: bool: default: false - } } output: logs: { diff --git a/src/sources/journald.rs b/src/sources/journald.rs index 2a6c927ab500f..3b5f2c65f3b9a 100644 --- a/src/sources/journald.rs +++ b/src/sources/journald.rs @@ -75,8 +75,9 @@ pub struct JournaldConfig { pub data_dir: Option, pub batch_size: Option, pub journalctl_path: Option, + /// Deprecated #[serde(default)] - pub remap_priority: bool, + remap_priority: bool, } inventory::submit! { @@ -97,6 +98,10 @@ impl SourceConfig for JournaldConfig { shutdown: ShutdownSignal, out: Pipeline, ) -> crate::Result { + if self.remap_priority { + warn!("Option `remap_priority` has been deprecated. Please use the `remap` transform and function `to_syslog_level` instead."); + } + let data_dir = globals.resolve_and_make_data_subdir(self.data_dir.as_ref(), name)?; let include_units = match (!self.units.is_empty(), !self.include_units.is_empty()) { From 39428b1ed6f4b5d7d5efb7c496b930d135cede3e Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Thu, 14 Jan 2021 03:08:41 -0400 Subject: [PATCH 009/769] chore(ci): Add S3 redirects for old artifact names (#5988) This uploads artifact redirects as part of the release to keep backwards compatibility with the links. For example, https://packages.timber.io/vector/nightly/2021-01-10/vector-amd64.deb is pointed to https://packages.timber.io/vector/nightly/2021-01-10/vector-nightly-amd64.deb. Signed-off-by: Jesse Szwedko --- scripts/release-s3.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scripts/release-s3.sh b/scripts/release-s3.sh index ef355db9656b2..4edd7c579336c 100755 --- a/scripts/release-s3.sh +++ b/scripts/release-s3.sh @@ -65,6 +65,15 @@ if [[ "$CHANNEL" == "nightly" ]]; then aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read echo "Uploaded archives" + echo "Redirecting old artifact names" + find "$td_nightly" -maxdepth 1 -type f -print 0 | while read -r -d $'\0' file ; do + file=$(basename "$file") + # vector-nightly-amd64.deb -> vector-amd64.deb + echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/$DATE/${file/-nightly//}" --website-redirect "/vector/nightly/$DATE/$file" --acl public-read + echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/latest/${file/-nightly//}" --website-redirect "/vector/nightly/latest/$file" --acl public-read + done + echo "Redirected old artifact names" + # Verify that the files exist and can be downloaded echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" sleep "$VERIFY_TIMEOUT" @@ -101,6 +110,15 @@ elif [[ "$CHANNEL" == "latest" ]]; then aws s3 cp "$td_latest" "s3://packages.timber.io/vector/latest/" --recursive --sse --acl public-read echo "Uploaded latest archives" + echo "Redirecting old artifact names" + find "$td" -maxdepth 1 -type f -print 0 | while read -r -d $'\0' file ; do + file=$(basename "$file") + # vector-$version-amd64.deb -> vector-amd64.deb + echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/$i/${file/-$i//}" --website-redirect "/vector/$i/$file" --acl public-read + echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/latest/${file/-$i//}" --website-redirect "/vector/latest/$file" --acl public-read + done + echo "Redirected old artifact names" + # Verify that the files exist and can be downloaded sleep "$VERIFY_TIMEOUT" echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" From 19600da769c1aa969b96f238f79c1d4ed91a5d5d Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Thu, 14 Jan 2021 09:52:40 +0100 Subject: [PATCH 010/769] chore(remap): add VRL test harness (#6022) Signed-off-by: Jean Mertz --- Cargo.lock | 100 +++++++++++++++- Cargo.toml | 3 +- lib/remap-tests/Cargo.toml | 13 +++ lib/remap-tests/src/main.rs | 151 +++++++++++++++++++++++++ lib/remap-tests/tests/example.vrl | 6 + lib/remap-tests/tests/invalid_path.vrl | 11 ++ 6 files changed, 278 insertions(+), 6 deletions(-) create mode 100644 lib/remap-tests/Cargo.toml create mode 100644 lib/remap-tests/src/main.rs create mode 100644 lib/remap-tests/tests/example.vrl create mode 100644 lib/remap-tests/tests/invalid_path.vrl diff --git a/Cargo.lock b/Cargo.lock index 63e211446f55f..00beb07147784 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,6 +55,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" + [[package]] name = "ansi_term" version = "0.11.0" @@ -1637,6 +1643,17 @@ dependencies = [ "generic-array 0.14.4", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users 0.3.5", + "winapi 0.3.9", +] + [[package]] name = "dirs" version = "2.0.2" @@ -1754,6 +1771,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.26" @@ -4586,6 +4609,31 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "prettydiff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bc9e8bdfe446d34975ff774fbb4a2b944d17054f6b5845ec132d4fb9ff8f559" +dependencies = [ + "ansi_term 0.9.0", + "prettytable-rs", + "structopt 0.2.18", +] + +[[package]] +name = "prettytable-rs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" +dependencies = [ + "atty", + "csv", + "encode_unicode", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -5174,7 +5222,7 @@ dependencies = [ "remap-lang", "rustyline", "serde_json", - "structopt", + "structopt 0.3.21", "thiserror", ] @@ -5224,6 +5272,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "remap-tests" +version = "0.1.0" +dependencies = [ + "ansi_term 0.12.1", + "prettydiff", + "remap-functions", + "remap-lang", + "serde_json", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -5377,7 +5436,7 @@ checksum = "09ac05563f83489b19b4d413607a30821ab08bbd9007d14fa05618da3ef09d8b" dependencies = [ "async-trait", "chrono", - "dirs", + "dirs 2.0.2", "futures 0.3.9", "hyper", "pin-project 0.4.27", @@ -6306,6 +6365,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "structopt" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16c2cdbf9cc375f15d1b4141bc48aeef444806655cd0e904207edc8d68d86ed7" +dependencies = [ + "clap", + "structopt-derive 0.2.18", +] + [[package]] name = "structopt" version = "0.3.21" @@ -6314,7 +6383,19 @@ checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" dependencies = [ "clap", "lazy_static", - "structopt-derive", + "structopt-derive 0.4.14", +] + +[[package]] +name = "structopt-derive" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010261a84b37689f9ed7d395165029f9cc7abb9f56bbfe86bee2597ed25107" +dependencies = [ + "heck", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", ] [[package]] @@ -6457,6 +6538,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs 1.0.5", + "winapi 0.3.9", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -7440,7 +7532,7 @@ dependencies = [ "snap", "stream-cancel", "strip-ansi-escapes", - "structopt", + "structopt 0.3.21", "syslog", "syslog_loose", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 232213f44b748..7f4f39f3096ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,11 +42,10 @@ members = [ "lib/k8s-test-framework", "lib/portpicker", "lib/prometheus-parser", - "lib/vector-api-client", "lib/remap-cli", - "lib/remap-lang", "lib/remap-functions", "lib/remap-lang", + "lib/remap-tests", "lib/shared", "lib/tracing-limit", "lib/vector-api-client", diff --git a/lib/remap-tests/Cargo.toml b/lib/remap-tests/Cargo.toml new file mode 100644 index 0000000000000..1e1b730dfc2c0 --- /dev/null +++ b/lib/remap-tests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "remap-tests" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2018" +publish = false + +[dependencies] +ansi_term = "0.12" +prettydiff = "0.4" +remap = { package = "remap-lang", path = "../remap-lang" } +remap-functions = { path = "../remap-functions" } +serde_json = "1" diff --git a/lib/remap-tests/src/main.rs b/lib/remap-tests/src/main.rs new file mode 100644 index 0000000000000..c7c04b1cd424b --- /dev/null +++ b/lib/remap-tests/src/main.rs @@ -0,0 +1,151 @@ +use ansi_term::Colour; +use remap::{prelude::*, Program, Runtime}; +use std::fs; +use std::path::PathBuf; + +fn main() { + let mut failed_count = 0; + let tests = fs::read_dir("tests").expect("dir exists"); + + for file in tests { + let mut test = Test::new(file.expect("file").path()); + + print!("{:.<30}", test.name); + + if test.skip { + println!("{}", Colour::Yellow.bold().paint("SKIPPED")); + } + + let state = state::Program::default(); + let mut runtime = Runtime::new(state); + let program = Program::new(&test.source, &remap_functions::all(), None, true); + let want = test.result.to_string(); + + match program { + Ok(program) => { + let result = runtime.execute(&mut test.object, &program); + + match result { + Ok(value) => { + let got = value.to_string(); + + if got == want { + println!("{}", Colour::Green.bold().paint("OK")); + } else { + println!("{} (expectation)", Colour::Red.bold().paint("FAILED")); + + let diff = + prettydiff::diff_chars(&got, &want).set_highlight_whitespace(true); + println!(" {}", diff); + + failed_count += 1; + } + } + Err(err) => { + let got = err.to_string(); + if got == want { + println!("{}", Colour::Green.bold().paint("OK")); + } else { + println!("{} (runtime)", Colour::Red.bold().paint("FAILED")); + println!("{}", err); + } + } + } + } + Err(err) => { + let got = err.to_string().trim().to_owned(); + let want = want.trim().to_owned(); + if got == want { + println!("{}", Colour::Green.bold().paint("OK")); + } else { + println!("{} (compilation)", Colour::Red.bold().paint("FAILED")); + + let diff = prettydiff::diff_chars(&got, &want).set_highlight_whitespace(true); + println!("{}", diff); + } + } + } + } + + if failed_count > 0 { + std::process::exit(1) + } +} + +#[derive(Debug)] +struct Test { + name: String, + source: String, + object: Value, + result: String, + skip: bool, +} + +enum CaptureMode { + Result, + Object, + None, +} + +impl Test { + fn new(path: PathBuf) -> Self { + let name = path + .to_string_lossy() + .strip_prefix("tests/") + .expect("test") + .to_owned(); + + let content = fs::read_to_string(path).expect("content"); + + let mut source = String::new(); + let mut object = String::new(); + let mut result = String::new(); + let mut skip = false; + + if content.starts_with("# SKIP") { + skip = true; + } + + let mut capture_mode = CaptureMode::None; + for mut line in content.lines() { + if line.starts_with('#') { + line = line.strip_prefix('#').expect("prefix"); + line = line.strip_prefix(' ').unwrap_or(line); + + if line.starts_with("object:") { + capture_mode = CaptureMode::Object; + line = line.strip_prefix("object:").expect("object").trim_start(); + } else if line.starts_with("result:") { + capture_mode = CaptureMode::Result; + line = line.strip_prefix("result:").expect("result").trim_start(); + } + + match capture_mode { + CaptureMode::None => continue, + CaptureMode::Result => { + result.push_str(line); + result.push('\n'); + } + CaptureMode::Object => { + object.push_str(line); + } + } + } else { + source.push_str(line); + source.push('\n') + } + } + + let object = serde_json::from_str::<'_, Value>(&object).expect("valid object"); + + result = result.trim_end().to_owned(); + + Self { + name, + source, + object, + result, + skip, + } + } +} diff --git a/lib/remap-tests/tests/example.vrl b/lib/remap-tests/tests/example.vrl new file mode 100644 index 0000000000000..0b7c25c28ca9e --- /dev/null +++ b/lib/remap-tests/tests/example.vrl @@ -0,0 +1,6 @@ +# object: { "bar": "bar" } +# result: { "bar": "foo", "foo": "test" } + +.foo = "test" +.bar = "foo" +. diff --git a/lib/remap-tests/tests/invalid_path.vrl b/lib/remap-tests/tests/invalid_path.vrl new file mode 100644 index 0000000000000..f31de7adfa9d0 --- /dev/null +++ b/lib/remap-tests/tests/invalid_path.vrl @@ -0,0 +1,11 @@ +# object: { "bar": "bar" } +# result: +# +# remap error: parser error: --> 2:6 +# | +# 2 | .foo.␊ +# | ^--- +# | +# = expected path segment + +.foo. From 96e445fa190d3ec39b6a09dbb8104f5c828facce Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Thu, 14 Jan 2021 09:59:52 -0400 Subject: [PATCH 011/769] Revert "chore(ci): Add S3 redirects for old artifact names (#5988)" (#6038) This reverts commit 39428b1ed6f4b5d7d5efb7c496b930d135cede3e. Signed-off-by: Jesse Szwedko --- scripts/release-s3.sh | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/scripts/release-s3.sh b/scripts/release-s3.sh index 4edd7c579336c..ef355db9656b2 100755 --- a/scripts/release-s3.sh +++ b/scripts/release-s3.sh @@ -65,15 +65,6 @@ if [[ "$CHANNEL" == "nightly" ]]; then aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read echo "Uploaded archives" - echo "Redirecting old artifact names" - find "$td_nightly" -maxdepth 1 -type f -print 0 | while read -r -d $'\0' file ; do - file=$(basename "$file") - # vector-nightly-amd64.deb -> vector-amd64.deb - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/$DATE/${file/-nightly//}" --website-redirect "/vector/nightly/$DATE/$file" --acl public-read - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/latest/${file/-nightly//}" --website-redirect "/vector/nightly/latest/$file" --acl public-read - done - echo "Redirected old artifact names" - # Verify that the files exist and can be downloaded echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" sleep "$VERIFY_TIMEOUT" @@ -110,15 +101,6 @@ elif [[ "$CHANNEL" == "latest" ]]; then aws s3 cp "$td_latest" "s3://packages.timber.io/vector/latest/" --recursive --sse --acl public-read echo "Uploaded latest archives" - echo "Redirecting old artifact names" - find "$td" -maxdepth 1 -type f -print 0 | while read -r -d $'\0' file ; do - file=$(basename "$file") - # vector-$version-amd64.deb -> vector-amd64.deb - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/$i/${file/-$i//}" --website-redirect "/vector/$i/$file" --acl public-read - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/latest/${file/-$i//}" --website-redirect "/vector/latest/$file" --acl public-read - done - echo "Redirected old artifact names" - # Verify that the files exist and can be downloaded sleep "$VERIFY_TIMEOUT" echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" From 281db2bb0b09b7851e624065be27ae9c4a992a63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thor=20Anker=20Kvisg=C3=A5rd=20Lange?= Date: Thu, 14 Jan 2021 15:19:47 +0100 Subject: [PATCH 012/769] fix(kubernetes platform): Adjusting to adhere to linting rules used by ct (#6021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thor Anker Kvisgård Lange --- distribution/helm/vector-agent/Chart.yaml | 4 ++-- distribution/helm/vector-agent/values.yaml | 6 +++--- distribution/helm/vector-aggregator/Chart.yaml | 4 ++-- distribution/helm/vector-shared/values.yaml | 0 distribution/helm/vector/Chart.yaml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 distribution/helm/vector-shared/values.yaml diff --git a/distribution/helm/vector-agent/Chart.yaml b/distribution/helm/vector-agent/Chart.yaml index 0bed00cd426ea..0c46dbf289f3c 100644 --- a/distribution/helm/vector-agent/Chart.yaml +++ b/distribution/helm/vector-agent/Chart.yaml @@ -3,8 +3,8 @@ name: vector-agent type: application icon: https://vector.dev/press/vector-icon.svg description: A Helm chart to collect Kubernetes logs with Vector -version: "0.0.0" # overwritten via --version on publishing -appVersion: "0.0.0" # overwritten via --app-version on publishing +version: "0.0.0" # overwritten via --version on publishing +appVersion: "0.0.0" # overwritten via --app-version on publishing home: https://vector.dev/ sources: - https://github.com/timberio/vector/ diff --git a/distribution/helm/vector-agent/values.yaml b/distribution/helm/vector-agent/values.yaml index 6d074b54cc6a0..3e018b6eb513e 100644 --- a/distribution/helm/vector-agent/values.yaml +++ b/distribution/helm/vector-agent/values.yaml @@ -138,11 +138,11 @@ vectorSink: # The name to use for the "built-in" vector sink. sinkId: vector_sink # Inputs of the built-in vector sink. - inputs: null # ["my_input_1", "my_input_2"] + inputs: null # ["my_input_1", "my_input_2"] # The host of the Vector to send the data to. - host: null # "vector.internal" + host: null # "vector.internal" # The port of the Vector to send the data to. - port: null # 9000 + port: null # 9000 # Raw TOML config to embed at the vector sink. rawConfig: null diff --git a/distribution/helm/vector-aggregator/Chart.yaml b/distribution/helm/vector-aggregator/Chart.yaml index e01c5f7a75dda..6ef6b60c32543 100644 --- a/distribution/helm/vector-aggregator/Chart.yaml +++ b/distribution/helm/vector-aggregator/Chart.yaml @@ -3,8 +3,8 @@ name: vector-aggregator type: application icon: https://vector.dev/press/vector-icon.svg description: A Helm chart to aggregate data with Vector -version: "0.0.0" # overwritten via --version on publishing -appVersion: "0.0.0" # overwritten via --app-version on publishing +version: "0.0.0" # overwritten via --version on publishing +appVersion: "0.0.0" # overwritten via --app-version on publishing home: https://vector.dev/ sources: - https://github.com/timberio/vector/ diff --git a/distribution/helm/vector-shared/values.yaml b/distribution/helm/vector-shared/values.yaml new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/distribution/helm/vector/Chart.yaml b/distribution/helm/vector/Chart.yaml index d4d1e7caa0b3f..828632ee62f66 100644 --- a/distribution/helm/vector/Chart.yaml +++ b/distribution/helm/vector/Chart.yaml @@ -3,8 +3,8 @@ name: vector type: application icon: https://vector.dev/press/vector-icon.svg description: A Helm chart for Vector observability stack -version: "0.0.0" # overwritten via --version on publishing -appVersion: "0.0.0" # overwritten via --app-version on publishing +version: "0.0.0" # overwritten via --version on publishing +appVersion: "0.0.0" # overwritten via --app-version on publishing home: https://vector.dev/ sources: - https://github.com/timberio/vector/ From 36adac70be243988803f9fe10580f06a8818cf3c Mon Sep 17 00:00:00 2001 From: FungusHumungus Date: Thu, 14 Jan 2021 14:57:13 +0000 Subject: [PATCH 013/769] fix(remap): allow comments between multiline statements (#6036) * Allow comments between multiline statements Signed-off-by: Stephen Wakely * Remove trailing spaces Signed-off-by: Stephen Wakely * Add test for empty line without comment in multiline expression Signed-off-by: Stephen Wakely --- lib/remap-lang/grammar.pest | 4 +++- lib/remap-lang/src/error.rs | 1 + tests/behavior/transforms/remap.toml | 33 ++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/remap-lang/grammar.pest b/lib/remap-lang/grammar.pest index 55bf4d0474b35..a50654ddf497f 100644 --- a/lib/remap-lang/grammar.pest +++ b/lib/remap-lang/grammar.pest @@ -136,7 +136,9 @@ reserved_keyword = @{ COMMENT = _{ "#" ~ (!NEWLINE ~ ANY)* } -WHITESPACE = _{ " " | "\t" | ("\\" ~ NEWLINE) } +empty_line = _{ WHITESPACE* ~ NEWLINE | WHITESPACE* ~ COMMENT } + +WHITESPACE = _{ " " | "\t" | ("\\" ~ empty_line* ) } // end of expression EOE = _{ NEWLINE | ";" } diff --git a/lib/remap-lang/src/error.rs b/lib/remap-lang/src/error.rs index d792423c7bfca..afcc68523e451 100644 --- a/lib/remap-lang/src/error.rs +++ b/lib/remap-lang/src/error.rs @@ -92,6 +92,7 @@ impl fmt::Display for Rule { comparison, EOE: "", EOI: "", + empty_line, equality, expression, expressions, diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index 7635415594513..fafb3790582d3 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -1698,10 +1698,17 @@ [transforms.remap_comments] inputs = [] type = "remap" - source = """ + source = ''' .a = 1 # .a = 2 # .a = 3 - """ + + .b = .a == 2 || \ + + # We should ignore this too. + # .a == 3 && \ + + .a == 1 + ''' [[tests]] name = "remap_comments" [tests.input] @@ -1712,3 +1719,25 @@ extract_from = "remap_comments" [[tests.outputs.conditions]] "a.equals" = 1 + "b.equals" = true + +[transforms.remap_multiline] + inputs = [] + type = "remap" + source = ''' + .a = "A long " + \ + + "multiline " + \ + "string" + ''' +[[tests]] + name = "remap_multiline" + [tests.input] + insert_at = "remap_multiline" + type = "log" + [tests.input.log_fields] + [[tests.outputs]] + extract_from = "remap_multiline" + [[tests.outputs.conditions]] + type = "remap" + source = '.a == "A long multiline string"' From eac9388abba1f1bb6116ae7fd5d3e84d553be906 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Thu, 14 Jan 2021 15:30:34 -0400 Subject: [PATCH 014/769] chore(ci): Pin Ruby at 2.7 in CI for OSX (#6055) It was installing the now-released 3.0 Signed-off-by: Jesse Szwedko --- scripts/environment/bootstrap-macos-10.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/environment/bootstrap-macos-10.sh b/scripts/environment/bootstrap-macos-10.sh index 5455204b701a9..82676ad5a786b 100644 --- a/scripts/environment/bootstrap-macos-10.sh +++ b/scripts/environment/bootstrap-macos-10.sh @@ -1,7 +1,7 @@ #! /usr/bin/env bash set -e -o verbose -brew install ruby coreutils cuelang/tap/cue +brew install ruby@2.7 coreutils cuelang/tap/cue echo "export PATH=\"/usr/local/opt/ruby/bin:\$PATH\"" >> "$HOME/.bash_profile" From 50ec1abe557ea78514da64aa2f7af2d6562eb988 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 14 Jan 2021 22:44:02 +0300 Subject: [PATCH 015/769] enhancement(kubernetes platform): Allow passing component configs as YAML in Helm charts (#5673) * Add a snapshot testing mechansim for Helm charts Signed-off-by: MOZGIII * Exclude helm snapshot YAMLs from style checks Signed-off-by: MOZGIII * Allow arbitrarily nested snapshot directories Signed-off-by: MOZGIII * Run Helm with --debug at snapshot tester Signed-off-by: MOZGIII * Implement TOML templating Signed-off-by: MOZGIII * Add first helm shapshot tests Signed-off-by: MOZGIII * Register check-helm-snapshots and update-helm-snapshots at Makefile Signed-off-by: MOZGIII * Add make check-helm-snapshots to CI Signed-off-by: MOZGIII * Mark helm snapshots as generated files Signed-off-by: MOZGIII * Update generated Kubernetes YAMLs Signed-off-by: MOZGIII * Remove a pointless comment for data_dir Signed-off-by: MOZGIII * Update Kubernetes YAMLs and Helm snapshots Signed-off-by: MOZGIII * Correct the comment Signed-off-by: MOZGIII * Fix the rawConfig embedding Signed-off-by: MOZGIII * Update helm snapshots Signed-off-by: MOZGIII * Adjust E2E tests Signed-off-by: MOZGIII --- .gitattributes | 2 + .github/workflows/test.yml | 3 + Makefile | 11 +- .../vector-agent/templates/configmap.yaml | 91 ++---- distribution/helm/vector-agent/values.yaml | 28 +- .../templates/configmap.yaml | 59 +--- .../helm/vector-aggregator/values.yaml | 15 +- .../helm/vector-shared/templates/_metrics.tpl | 32 +- .../templates/_vector_config.tpl | 104 +++++++ .../kubernetes/vector-agent/resources.yaml | 12 +- .../vector-aggregator/resources.yaml | 13 +- .../kubernetes/vector-all/resources.yaml | 30 +- lib/k8s-e2e-tests/tests/vector-agent.rs | 93 ++++++ lib/k8s-e2e-tests/tests/vector-aggregator.rs | 12 +- lib/k8s-e2e-tests/tests/vector.rs | 5 +- scripts/check-style.sh | 1 + scripts/helm-template-snapshot.sh | 101 +++++++ tests/helm-snapshots/README.md | 5 + .../builtin_configs/vector-agent/config.sh | 5 + .../vector-agent/snapshot.yaml | 240 +++++++++++++++ .../builtin_configs/vector-agent/values.yaml | 38 +++ .../vector-aggregator/config.sh | 5 + .../vector-aggregator/snapshot.yaml | 189 ++++++++++++ .../vector-aggregator/values.yaml | 20 ++ .../topology_config/vector-agent/config.sh | 5 + .../vector-agent/snapshot.yaml | 280 ++++++++++++++++++ .../topology_config/vector-agent/values.yaml | 59 ++++ .../vector-aggregator/config.sh | 5 + .../vector-aggregator/snapshot.yaml | 241 +++++++++++++++ .../vector-aggregator/values.yaml | 59 ++++ 30 files changed, 1593 insertions(+), 170 deletions(-) create mode 100644 distribution/helm/vector-shared/templates/_vector_config.tpl create mode 100755 scripts/helm-template-snapshot.sh create mode 100644 tests/helm-snapshots/README.md create mode 100644 tests/helm-snapshots/builtin_configs/vector-agent/config.sh create mode 100644 tests/helm-snapshots/builtin_configs/vector-agent/snapshot.yaml create mode 100644 tests/helm-snapshots/builtin_configs/vector-agent/values.yaml create mode 100644 tests/helm-snapshots/builtin_configs/vector-aggregator/config.sh create mode 100644 tests/helm-snapshots/builtin_configs/vector-aggregator/snapshot.yaml create mode 100644 tests/helm-snapshots/builtin_configs/vector-aggregator/values.yaml create mode 100644 tests/helm-snapshots/topology_config/vector-agent/config.sh create mode 100644 tests/helm-snapshots/topology_config/vector-agent/snapshot.yaml create mode 100644 tests/helm-snapshots/topology_config/vector-agent/values.yaml create mode 100644 tests/helm-snapshots/topology_config/vector-aggregator/config.sh create mode 100644 tests/helm-snapshots/topology_config/vector-aggregator/snapshot.yaml create mode 100644 tests/helm-snapshots/topology_config/vector-aggregator/values.yaml diff --git a/.gitattributes b/.gitattributes index d3380edf149c5..34df3ae250fae 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,5 @@ scripts/* linguist-detectable=false Makefile linguist-detectable=false distribution/helm/*/charts/*.tgz filter=lfs diff=lfs merge=lfs -text + +tests/helm-snapshots/**/snapshot.yaml linguist-generated=true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1eb440a3c3237..5a0e2dd2b927b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -253,6 +253,9 @@ jobs: - name: Check helm lint if: needs.changes.outputs.helm == 'true' run: make check-helm-lint + - name: Check that Helm template snapshots don't diverge from Helm + if: needs.changes.outputs.helm == 'true' + run: make check-helm-snapshots - name: Check that generated Kubernetes YAMLs don't diverge from Helm if: needs.changes.outputs.helm == 'true' run: make check-kubernetes-yaml diff --git a/Makefile b/Makefile index 97d541eaccd5d..debd3dee6f879 100644 --- a/Makefile +++ b/Makefile @@ -550,7 +550,8 @@ check-all: ## Check everything check-all: check-fmt check-clippy check-style check-markdown check-docs check-all: check-version check-examples check-component-features check-all: check-scripts -check-all: check-helm-lint check-helm-dependencies check-kubernetes-yaml +check-all: check-helm-lint check-helm-dependencies check-helm-snapshots +check-all: check-kubernetes-yaml .PHONY: check-component-features check-component-features: ## Check that all component features are setup properly @@ -596,6 +597,10 @@ check-helm-lint: ## Check that Helm charts pass helm lint check-helm-dependencies: ## Check that Helm charts have up-to-date dependencies ${MAYBE_ENVIRONMENT_EXEC} ./scripts/helm-dependencies.sh validate +.PHONY: check-helm-snapshots +check-helm-snapshots: ## Check that the Helm template snapshots do not diverge from the Helm charts + ${MAYBE_ENVIRONMENT_EXEC} ./scripts/helm-template-snapshot.sh check + .PHONY: check-kubernetes-yaml check-kubernetes-yaml: ## Check that the generated Kubernetes YAML configs are up to date ${MAYBE_ENVIRONMENT_EXEC} ./scripts/kubernetes-yaml.sh check @@ -787,6 +792,10 @@ git-hooks: ## Add Vector-local git hooks for commit sign-off update-helm-dependencies: ## Recursively update the dependencies of the Helm charts in the proper order ${MAYBE_ENVIRONMENT_EXEC} ./scripts/helm-dependencies.sh update +.PHONY: update-helm-snapshots +update-helm-snapshots: ## Update the Helm template snapshots from the Helm charts + ${MAYBE_ENVIRONMENT_EXEC} ./scripts/helm-template-snapshot.sh update + .PHONY: update-kubernetes-yaml update-kubernetes-yaml: ## Regenerate the Kubernetes YAML configs ${MAYBE_ENVIRONMENT_EXEC} ./scripts/kubernetes-yaml.sh update diff --git a/distribution/helm/vector-agent/templates/configmap.yaml b/distribution/helm/vector-agent/templates/configmap.yaml index eb344e7cbe5c6..3d409bb4d7f0b 100644 --- a/distribution/helm/vector-agent/templates/configmap.yaml +++ b/distribution/helm/vector-agent/templates/configmap.yaml @@ -9,40 +9,28 @@ data: # We leave `vector.toml` file name available to let externally managed config # maps to provide it. managed.toml: | - # Configuration for vector. - # Docs: https://vector.dev/docs/ + {{- include "libvector.vectorConfigHeader" . | nindent 4 -}} - # Data dir is location controlled at the `DaemonSet`. - data_dir = "{{ .Values.globalOptions.dataDir }}" - - {{- with .Values.logSchema }} - [log_schema] - host_key = "{{ .hostKey }}" - message_key = "{{ .messageKey }}" - source_type_key = "{{ .sourceTypeKey }}" - timestamp_key = "{{ .timestampKey }}" - {{- end }} - - {{- if .Values.kubernetesLogsSource.enabled }} + {{- with .Values.kubernetesLogsSource }} + {{- if .enabled }} # Ingest logs from Kubernetes. - [sources.{{ .Values.kubernetesLogsSource.sourceId }}] - type = "kubernetes_logs" - - {{- with .Values.kubernetesLogsSource.rawConfig }} - {{- . | nindent 6 }} - {{- end }} + {{- $value := merge (dict) .config -}} + {{- $_ := set $value "type" "kubernetes_logs" -}} + {{- $_ := set $value "rawConfig" .rawConfig -}} + {{- tuple .sourceId $value | include "libvector.vectorSourceConfig" | nindent 4 -}} + {{- end }} {{- end }} - {{- if .Values.vectorSink.enabled }} + {{- with .Values.vectorSink -}} + {{- if .enabled }} # Send logs to the aggregator. - [sinks.{{ .Values.vectorSink.sinkId }}] - type = "vector" - inputs = {{ required "You must specify the `inputs` for the built-in vector sink" .Values.vectorSink.inputs | toJson }} - address = "{{ include "vector-agent.vectorSinkAddress" . }}" - - {{- with .Values.vectorSink.rawConfig }} - {{- . | nindent 6 }} - {{- end }} + {{- $value := merge (dict) .config -}} + {{- $_ := set $value "type" "vector" -}} + {{- $_ := set $value "inputs" (required "You must specify the `inputs` for the built-in vector sink" .inputs) -}} + {{- $_ := set $value "address" (include "vector-agent.vectorSinkAddress" $) -}} + {{- $_ := set $value "rawConfig" .rawConfig -}} + {{- tuple .sinkId $value | include "libvector.vectorSinkConfig" | nindent 4 -}} + {{- end }} {{- end }} {{- $prometheusInputs := (list) -}} @@ -50,44 +38,15 @@ data: {{- if .enabled }} {{- $prometheusInputs = prepend $prometheusInputs .sourceId }} # Capture the metrics from the host. - [sources.{{ .sourceId }}] - type = "host_metrics" - - {{- with .rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} - {{- end }} - - {{- merge . (dict "prometheusInputs" $prometheusInputs) | include "libvector.metricsConfigPartial" | nindent 4 }} - - {{- range $sourceId, $source := .Values.sources }} - [sources.{{ $sourceId }}] - type = "{{ $source.type }}" - - {{- with $source.rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} - - {{- range $transformId, $transform := .Values.transforms }} - [transforms.{{ $transformId }}] - type = "{{ $transform.type }}" - inputs = {{ $transform.inputs | toJson }} + {{- $value := merge (dict) .config -}} + {{- $_ := set $value "type" "host_metrics" -}} + {{- $_ := set $value "rawConfig" .rawConfig -}} + {{- tuple .sourceId $value | include "libvector.vectorSourceConfig" | nindent 4 -}} + {{- end -}} + {{- end -}} - {{- with $transform.rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} + {{- merge . (dict "prometheusInputs" $prometheusInputs) | include "libvector.metricsConfigPartial" | nindent 4 -}} - {{- range $sinkId, $sink := .Values.sinks }} - [sinks.{{ $sinkId }}] - type = "{{ $sink.type }}" - inputs = {{ $sink.inputs | toJson }} - - {{- with $sink.rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} + {{- include "libvector.vectorTopology" .Values | nindent 4 -}} {{- end }} diff --git a/distribution/helm/vector-agent/values.yaml b/distribution/helm/vector-agent/values.yaml index 3e018b6eb513e..7680691dcbdc6 100644 --- a/distribution/helm/vector-agent/values.yaml +++ b/distribution/helm/vector-agent/values.yaml @@ -126,7 +126,10 @@ kubernetesLogsSource: enabled: true # The name to use for the "built-in" kubernetes logs source. sourceId: kubernetes_logs - # Raw TOML config to embed at the kubernetes logs source. + # Additional config to embed at the kubernetes logs source. + config: {} + # option: "value" + # Raw TOML config to embed at the kubernetes logs source (deprecated). rawConfig: null # The "built-in" vector sink, to send the logs to the vector aggregator. @@ -143,7 +146,10 @@ vectorSink: host: null # "vector.internal" # The port of the Vector to send the data to. port: null # 9000 - # Raw TOML config to embed at the vector sink. + # Additional config to embed at the vector sink. + config: {} + # option: "value" + # Raw TOML config to embed at the vector sink (deprecated). rawConfig: null # The "built-in" internal metrics source emitting Vector's internal opertaional @@ -153,7 +159,10 @@ internalMetricsSource: enabled: true # The name to use for the "built-in" internal metrics source. sourceId: internal_metrics - # Raw TOML config to embed at the internal metrics source. + # Additional config to embed at the internal metrics source. + config: {} + # option: "value" + # Raw TOML config to embed at the internal metrics source (deprecated). rawConfig: null # The "built-in" host metrics source emitting the metrics gathered from the node @@ -163,7 +172,10 @@ hostMetricsSource: enabled: true # The name to use for the "built-in" host metrics source. sourceId: host_metrics - # Raw TOML config to embed at the host metrics source. + # Additional config to embed at the host metrics source. + config: {} + # option: "value" + # Raw TOML config to embed at the host metrics source (deprecated). rawConfig: null # The "built-in" prometheus sink exposing metrics in the Prometheus scraping @@ -189,7 +201,10 @@ prometheusSink: listenAddress: "0.0.0.0" # The port to listen at. listenPort: "9090" - # Raw TOML config to embed at the vector source. + # Additional config to embed at the prometheus sink. + config: {} + # option: "value" + # Raw TOML config to embed at the prometheus sink (deprecated). rawConfig: null # Add Prometheus annotations to Pod to opt-in for Prometheus scraping. # To be used in clusters that rely on Pod annotations in the form of @@ -212,6 +227,7 @@ prometheusSink: sources: {} # source_name: # type: "source_type" + # option: "value" # rawConfig: | # option = "value" @@ -220,6 +236,7 @@ transforms: {} # transform_name: # type: "transform_type" # inputs: ["input1", "input2"] + # option: "value" # rawConfig: | # option = "value" @@ -228,5 +245,6 @@ sinks: {} # sink_name: # type: "sink_type" # inputs: ["input1", "input2"] + # option: "value" # rawConfig: | # option = "value" diff --git a/distribution/helm/vector-aggregator/templates/configmap.yaml b/distribution/helm/vector-aggregator/templates/configmap.yaml index f63e5a52a28f3..2f3ce4b84c39d 100644 --- a/distribution/helm/vector-aggregator/templates/configmap.yaml +++ b/distribution/helm/vector-aggregator/templates/configmap.yaml @@ -9,60 +9,21 @@ data: # We leave `vector.toml` file name available to let externally managed config # maps to provide it. managed.toml: | - # Configuration for vector. - # Docs: https://vector.dev/docs/ + {{- include "libvector.vectorConfigHeader" . | nindent 4 -}} - # Data dir is location controlled at the `StatefulSet`. - data_dir = "{{ .Values.globalOptions.dataDir }}" - - {{- with .Values.logSchema }} - [log_schema] - host_key = "{{ .hostKey }}" - message_key = "{{ .messageKey }}" - source_type_key = "{{ .sourceTypeKey }}" - timestamp_key = "{{ .timestampKey }}" - {{- end }} - - {{- if .Values.vectorSource.enabled }} + {{- with .Values.vectorSource }} + {{- if .enabled }} # Accept logs from Vector agents. - [sources.{{ .Values.vectorSource.sourceId }}] - type = "vector" - address = "{{ .Values.vectorSource.listenAddress }}:{{ .Values.vectorSource.listenPort }}" - - {{- with .Values.vectorSource.rawConfig }} - {{- . | nindent 6 }} - {{- end }} + {{- $value := merge (dict) .config -}} + {{- $_ := set $value "type" "vector" -}} + {{- $_ := set $value "address" (printf "%v:%v" .listenAddress .listenPort) -}} + {{- $_ := set $value "rawConfig" .rawConfig -}} + {{- tuple .sourceId $value | include "libvector.vectorSourceConfig" | nindent 4 -}} {{- end }} - - {{- include "libvector.metricsConfigPartial" . | nindent 4 }} - - {{- range $sourceId, $source := .Values.sources }} - [sources.{{ $sourceId }}] - type = "{{ $source.type }}" - - {{- with $source.rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} - - {{- range $transformId, $transform := .Values.transforms }} - [transforms.{{ $transformId }}] - type = "{{ $transform.type }}" - inputs = {{ $transform.inputs | toJson }} - - {{- with $transform.rawConfig }} - {{- . | nindent 6 }} - {{- end }} {{- end }} - {{- range $sinkId, $sink := .Values.sinks }} - [sinks.{{ $sinkId }}] - type = "{{ $sink.type }}" - inputs = {{ $sink.inputs | toJson }} + {{- include "libvector.metricsConfigPartial" . | nindent 4 }} - {{- with $sink.rawConfig }} - {{- . | nindent 6 }} - {{- end }} - {{- end }} + {{- include "libvector.vectorTopology" .Values | nindent 4 -}} {{- end }} diff --git a/distribution/helm/vector-aggregator/values.yaml b/distribution/helm/vector-aggregator/values.yaml index 167cfe6c8913d..a1bae3160c292 100644 --- a/distribution/helm/vector-aggregator/values.yaml +++ b/distribution/helm/vector-aggregator/values.yaml @@ -186,7 +186,10 @@ vectorSource: listenAddress: "0.0.0.0" # The port to listen at. listenPort: "9000" - # Raw TOML config to embed at the vector source. + # Additional config to embed at the vector source. + config: {} + # option: "value" + # Raw TOML config to embed at the vector source (deprecated). rawConfig: null # The "built-in" internal metrics source emitting Vector's internal opertaional @@ -196,7 +199,10 @@ internalMetricsSource: enabled: true # The name to use for the "built-in" internal metrics source. sourceId: internal_metrics - # Raw TOML config to embed at the internal metrics source. + # Additional config to embed at the internal metrics source. + config: {} + # option: "value" + # Raw TOML config to embed at the internal metrics source (deprecated). rawConfig: null # The "built-in" prometheus sink exposing metrics in the Prometheus scraping @@ -222,7 +228,10 @@ prometheusSink: listenAddress: "0.0.0.0" # The port to listen at. listenPort: "9090" - # Raw TOML config to embed at the vector source. + # Additional config to embed at the prometheus sink. + config: {} + # option: "value" + # Raw TOML config to embed at the prometheus sink (deprecated). rawConfig: null # Add Prometheus annotations to Pod to opt-in for Prometheus scraping. # To be used in clusters that rely on Pod annotations in the form of diff --git a/distribution/helm/vector-shared/templates/_metrics.tpl b/distribution/helm/vector-shared/templates/_metrics.tpl index 1153c353edf74..bfae1ed8b24f6 100644 --- a/distribution/helm/vector-shared/templates/_metrics.tpl +++ b/distribution/helm/vector-shared/templates/_metrics.tpl @@ -5,40 +5,40 @@ Common Vector configuration partial containing built-in metrics pipeline. Internal metrics are common, so we share and reuse the definition. */}} {{- define "libvector.metricsConfigPartial" -}} + {{- $values := .Values -}} + {{- $prometheusInputs := .prometheusInputs -}} {{- with $values.internalMetricsSource }} {{- if .enabled }} # Emit internal Vector metrics. -[sources.{{ .sourceId }}] - type = "internal_metrics" - - {{- with .rawConfig }} - {{- . | nindent 6 }} - {{- end }} +{{- $value := merge (dict) .config -}} +{{- $_ := set $value "type" "internal_metrics" -}} +{{- $_ := set $value "rawConfig" .rawConfig -}} +{{- tuple .sourceId $value | include "libvector.vectorSourceConfig" | nindent 0 -}} {{- end }} {{- end }} {{- with $values.prometheusSink }} {{- if .enabled }} + {{- $inputs := .inputs }} {{- if $prometheusInputs -}} -{{- $inputs = concat $inputs $prometheusInputs }} +{{- $inputs = concat $inputs $prometheusInputs }} {{- end }} {{- if and $values.internalMetricsSource.enabled (not .excludeInternalMetrics) -}} -{{- $inputs = prepend $inputs $values.internalMetricsSource.sourceId }} +{{- $inputs = prepend $inputs $values.internalMetricsSource.sourceId }} {{- end }} # Expose metrics for scraping in the Prometheus format. -[sinks.{{ .sinkId }}] - type = "prometheus" - inputs = {{ $inputs | toJson }} - address = "{{ .listenAddress }}:{{ .listenPort }}" - - {{- with .rawConfig }} - {{- . | nindent 6 }} - {{- end }} +{{- $value := merge (dict) .config -}} +{{- $_ := set $value "type" "prometheus" -}} +{{- $_ := set $value "inputs" $inputs -}} +{{- $_ := set $value "address" (printf "%v:%v" .listenAddress .listenPort) -}} +{{- $_ := set $value "rawConfig" .rawConfig -}} +{{- tuple .sinkId $value | include "libvector.vectorSinkConfig" | nindent 0 -}} {{- end }} {{- end }} + {{- end }} {{/* diff --git a/distribution/helm/vector-shared/templates/_vector_config.tpl b/distribution/helm/vector-shared/templates/_vector_config.tpl new file mode 100644 index 0000000000000..ee6a752610c75 --- /dev/null +++ b/distribution/helm/vector-shared/templates/_vector_config.tpl @@ -0,0 +1,104 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Serialize the passed Vector component configuration bits as TOML. +*/}} +{{- define "libvector.vectorComponentConfig" -}} +{{- $componentGroup := index . 0 -}} +{{- $componentId := index . 1 -}} +{{- $value := index . 2 -}} + +{{- $rawConfig := $value.rawConfig -}} +{{- $value = unset $value "rawConfig" -}} + +{{- $header := printf "[%s.%s]" $componentGroup $componentId -}} + +{{- /* Build the right hierarchy and evaluate the TOML. */ -}} +{{- $toml := toToml (dict $componentGroup (dict $componentId $value)) -}} +{{- /* Cut the root-level key containing the component kind name (i.e. `[sinks]`). */ -}} +{{- $toml = $toml | trimPrefix (printf "[%s]\n" $componentGroup) -}} +{{- /* Remove one level of indentation. */ -}} +{{- $toml = regexReplaceAllLiteral "(?m)^ " $toml "" -}} +{{- /* Cut tailing newline. */ -}} +{{- $toml = $toml | trimSuffix "\n" -}} +{{- /* Print the value. */ -}} +{{- $toml -}} + +{{- with $rawConfig -}} +{{- /* Here is a poor attempt to ensure raw config section is put under the */ -}} +{{- /* component-level section. What we're trying to do here is prohibited */ -}} +{{- /* in the TOML spec, but it may work in the simple case - and this is */ -}} +{{- /* what we have to support for the backward compatibility. */ -}} +{{- if contains (printf "[%s.%s." $componentGroup $componentId) $toml -}} +{{- $header| nindent 0 -}} +{{- end -}} +{{- /* Print the raw config. */ -}} + {{- $rawConfig | nindent 2 -}} +{{- end }} + +{{- printf "\n" -}} +{{- end }} + +{{/* +Serialize the passed Vector source configuration bits as TOML. +*/}} +{{- define "libvector.vectorSourceConfig" -}} +{{- $componentId := index . 0 -}} +{{- $value := index . 1 -}} +{{- tuple "sources" $componentId $value | include "libvector.vectorComponentConfig" -}} +{{- end }} + +{{/* +Serialize the passed Vector transform configuration bits as TOML. +*/}} +{{- define "libvector.vectorTransformConfig" -}} +{{- $componentId := index . 0 -}} +{{- $value := index . 1 -}} +{{- tuple "transforms" $componentId $value | include "libvector.vectorComponentConfig" -}} +{{- end }} + +{{/* +Serialize the passed Vector sink configuration bits as TOML. +*/}} +{{- define "libvector.vectorSinkConfig" -}} +{{- $componentId := index . 0 -}} +{{- $value := index . 1 -}} +{{- tuple "sinks" $componentId $value | include "libvector.vectorComponentConfig" -}} +{{- end }} + +{{/* +Serialize the passed Vector topology configuration bits as TOML. +*/}} +{{- define "libvector.vectorTopology" -}} +{{- range $componentId, $value := .sources }} +{{- tuple $componentId $value | include "libvector.vectorSourceConfig" | nindent 0 -}} +{{- end }} + +{{- range $componentId, $value := .transforms }} +{{- tuple $componentId $value | include "libvector.vectorTransformConfig" | nindent 0 -}} +{{- end }} + +{{- range $componentId, $value := .sinks }} +{{- tuple $componentId $value | include "libvector.vectorSinkConfig" | nindent 0 -}} +{{- end }} +{{- end }} + +{{/* +The common header for Vector ConfigMaps. +*/}} +{{- define "libvector.vectorConfigHeader" -}} +# Configuration for vector. +# Docs: https://vector.dev/docs/ + +data_dir = "{{ .Values.globalOptions.dataDir }}" +{{- printf "\n" -}} + +{{- with .Values.logSchema }} +[log_schema] + host_key = "{{ .hostKey }}" + message_key = "{{ .messageKey }}" + source_type_key = "{{ .sourceTypeKey }}" + timestamp_key = "{{ .timestampKey }}" + {{- printf "\n" -}} +{{- end }} +{{- end }} diff --git a/distribution/kubernetes/vector-agent/resources.yaml b/distribution/kubernetes/vector-agent/resources.yaml index fcfe9f12e0494..11505bf4cf2ad 100644 --- a/distribution/kubernetes/vector-agent/resources.yaml +++ b/distribution/kubernetes/vector-agent/resources.yaml @@ -35,29 +35,33 @@ data: managed.toml: | # Configuration for vector. # Docs: https://vector.dev/docs/ - - # Data dir is location controlled at the `DaemonSet`. + data_dir = "/vector-data-dir" + [log_schema] host_key = "host" message_key = "message" source_type_key = "source_type" timestamp_key = "timestamp" + # Ingest logs from Kubernetes. [sources.kubernetes_logs] type = "kubernetes_logs" + # Capture the metrics from the host. [sources.host_metrics] type = "host_metrics" + # Emit internal Vector metrics. [sources.internal_metrics] type = "internal_metrics" + # Expose metrics for scraping in the Prometheus format. [sinks.prometheus_sink] - type = "prometheus" - inputs = ["internal_metrics","host_metrics"] address = "0.0.0.0:9090" + inputs = ["internal_metrics", "host_metrics"] + type = "prometheus" --- # Source: vector-agent/templates/rbac.yaml # Permissions to use Kubernetes API. diff --git a/distribution/kubernetes/vector-aggregator/resources.yaml b/distribution/kubernetes/vector-aggregator/resources.yaml index f7a6bfad21a11..e3c92103ce1f0 100644 --- a/distribution/kubernetes/vector-aggregator/resources.yaml +++ b/distribution/kubernetes/vector-aggregator/resources.yaml @@ -35,27 +35,30 @@ data: managed.toml: | # Configuration for vector. # Docs: https://vector.dev/docs/ - - # Data dir is location controlled at the `StatefulSet`. + data_dir = "/vector-data-dir" + [log_schema] host_key = "host" message_key = "message" source_type_key = "source_type" timestamp_key = "timestamp" + # Accept logs from Vector agents. [sources.vector] - type = "vector" address = "0.0.0.0:9000" + type = "vector" + # Emit internal Vector metrics. [sources.internal_metrics] type = "internal_metrics" + # Expose metrics for scraping in the Prometheus format. [sinks.prometheus_sink] - type = "prometheus" - inputs = ["internal_metrics"] address = "0.0.0.0:9090" + inputs = ["internal_metrics"] + type = "prometheus" --- # Source: vector-aggregator/templates/service-headless.yaml apiVersion: v1 diff --git a/distribution/kubernetes/vector-all/resources.yaml b/distribution/kubernetes/vector-all/resources.yaml index 75324a2c0af19..5e23f0a2b5bb4 100644 --- a/distribution/kubernetes/vector-all/resources.yaml +++ b/distribution/kubernetes/vector-all/resources.yaml @@ -47,34 +47,39 @@ data: managed.toml: | # Configuration for vector. # Docs: https://vector.dev/docs/ - - # Data dir is location controlled at the `DaemonSet`. + data_dir = "/vector-data-dir" + [log_schema] host_key = "host" message_key = "message" source_type_key = "source_type" timestamp_key = "timestamp" + # Ingest logs from Kubernetes. [sources.kubernetes_logs] type = "kubernetes_logs" + # Send logs to the aggregator. [sinks.vector_sink] - type = "vector" - inputs = ["kubernetes_logs"] address = "vector-aggregator:9000" + inputs = ["kubernetes_logs"] + type = "vector" + # Capture the metrics from the host. [sources.host_metrics] type = "host_metrics" + # Emit internal Vector metrics. [sources.internal_metrics] type = "internal_metrics" + # Expose metrics for scraping in the Prometheus format. [sinks.prometheus_sink] - type = "prometheus" - inputs = ["internal_metrics","host_metrics"] address = "0.0.0.0:9090" + inputs = ["internal_metrics", "host_metrics"] + type = "prometheus" --- # Source: vector/charts/vector-aggregator/templates/configmap.yaml apiVersion: v1 @@ -93,27 +98,30 @@ data: managed.toml: | # Configuration for vector. # Docs: https://vector.dev/docs/ - - # Data dir is location controlled at the `StatefulSet`. + data_dir = "/vector-data-dir" + [log_schema] host_key = "host" message_key = "message" source_type_key = "source_type" timestamp_key = "timestamp" + # Accept logs from Vector agents. [sources.vector] - type = "vector" address = "0.0.0.0:9000" + type = "vector" + # Emit internal Vector metrics. [sources.internal_metrics] type = "internal_metrics" + # Expose metrics for scraping in the Prometheus format. [sinks.prometheus_sink] - type = "prometheus" - inputs = ["internal_metrics"] address = "0.0.0.0:9090" + inputs = ["internal_metrics"] + type = "prometheus" --- # Source: vector/charts/vector-agent/templates/rbac.yaml # Permissions to use Kubernetes API. diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 71c88bcad941e..92fffeaac7c26 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -9,6 +9,15 @@ use std::str::FromStr; const HELM_CHART_VECTOR_AGENT: &str = "vector-agent"; const HELM_VALUES_STDOUT_SINK: &str = r#" +sinks: + stdout: + type: "console" + inputs: ["kubernetes_logs"] + target: "stdout" + encoding: "json" +"#; + +const HELM_VALUES_STDOUT_SINK_RAW_CONFIG: &str = r#" sinks: stdout: type: "console" @@ -121,6 +130,90 @@ async fn simple() -> Result<(), Box> { Ok(()) } +/// This test validates that vector-agent picks up logs at the simplest case +/// possible - a new pod is deployed and prints to stdout, and we assert that +/// vector picks that up - but with the legacy `rawConfig` way of passing the +/// sink configuration. +#[tokio::test] +async fn simple_raw_config() -> Result<(), Box> { + let _guard = lock(); + let framework = make_framework(); + + let vector = framework + .vector( + "test-vector", + HELM_CHART_VECTOR_AGENT, + VectorConfig { + custom_helm_values: HELM_VALUES_STDOUT_SINK_RAW_CONFIG, + ..Default::default() + }, + ) + .await?; + framework + .wait_for_rollout( + "test-vector", + "daemonset/vector-agent", + vec!["--timeout=60s"], + ) + .await?; + + let test_namespace = framework.namespace("test-vector-test-pod").await?; + + let test_pod = framework + .test_pod(test_pod::Config::from_pod(&make_test_pod( + "test-vector-test-pod", + "test-pod", + "echo MARKER", + vec![], + vec![], + ))?) + .await?; + framework + .wait( + "test-vector-test-pod", + vec!["pods/test-pod"], + WaitFor::Condition("initialized"), + vec!["--timeout=60s"], + ) + .await?; + + let mut log_reader = framework.logs("test-vector", "daemonset/vector-agent")?; + smoke_check_first_line(&mut log_reader).await; + + // Read the rest of the log lines. + let mut got_marker = false; + look_for_log_line(&mut log_reader, |val| { + if val["kubernetes"]["pod_namespace"] != "test-vector-test-pod" { + // A log from something other than our test pod, pretend we don't + // see it. + return FlowControlCommand::GoOn; + } + + // Ensure we got the marker. + assert_eq!(val["message"], "MARKER"); + + if got_marker { + // We've already seen one marker! This is not good, we only emitted + // one. + panic!("Marker seen more than once"); + } + + // If we did, remember it. + got_marker = true; + + // Request to stop the flow. + FlowControlCommand::Terminate + }) + .await?; + + assert!(got_marker); + + drop(test_pod); + drop(test_namespace); + drop(vector); + Ok(()) +} + /// This test validates that vector-agent properly merges a log message that /// kubernetes has internally split into multiple partial log lines. #[tokio::test] diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index 17de7ce31bc1b..1d57a1031556f 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -7,18 +7,16 @@ const HELM_VALUES_DUMMY_TOPOLOGY: &str = r#" sources: dummy: type: "generator" - rawConfig: | - format = "shuffle" - lines = ["Hello world"] - interval = 60 # once a minute + format: "shuffle" + lines: ["Hello world"] + interval: 60 # once a minute sinks: stdout: type: "console" inputs: ["dummy"] - rawConfig: | - target = "stdout" - encoding = "json" + target: "stdout" + encoding: "json" "#; /// This test validates that vector-aggregator can deploy with the default diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index deaf159e29cac..7f847ce5b3cfc 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -14,9 +14,8 @@ vector-aggregator: stdout: type: "console" inputs: ["vector"] - rawConfig: | - target = "stdout" - encoding = "json" + target: "stdout" + encoding: "json" "#; /// This test validates that vector picks up logs with an agent and diff --git a/scripts/check-style.sh b/scripts/check-style.sh index 8e2941df575b6..995dc44e06678 100755 --- a/scripts/check-style.sh +++ b/scripts/check-style.sh @@ -34,6 +34,7 @@ for FILE in $(git ls-files); do *sig) continue;; tests/data*) continue;; distribution/kubernetes/*/*.yaml) continue;; + tests/helm-snapshots/*/snapshot.yaml) continue;; esac # Skip all directories (usually theis only happens when we have symlinks). diff --git a/scripts/helm-template-snapshot.sh b/scripts/helm-template-snapshot.sh new file mode 100755 index 0000000000000..298ebe859d1a8 --- /dev/null +++ b/scripts/helm-template-snapshot.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +# helm-template-snapshot.sh +# +# SUMMARY +# +# Manages the Helm template snapshots. +# See usage function in the code or run without arguments. + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +CONFIGURATIONS_DIR="tests/helm-snapshots" + +generate() { + local RELEASE_NAME="$1" + local CHART="$2" + local VALUES_FILE="$3" + + # Print header. + cat <"$TARGET_FILE" + ) + done +} + +check() { + list-config-files + for CONFIG_FILE in "${CONFIG_FILES[@]}"; do + VALUES_FILE="$(dirname "$CONFIG_FILE")/values.yaml" + TARGET_FILE="$(dirname "$CONFIG_FILE")/snapshot.yaml" + ( + # shellcheck disable=SC1090 + source "$CONFIG_FILE" + GENERATED="$(generate "$RELEASE_NAME" "$CHART" "$VALUES_FILE")" + FILE="$(cat "$TARGET_FILE")" + + if [[ "$GENERATED" != "$FILE" ]]; then + echo "Error: snapshot ($TARGET_FILE) does not match the generated version" >&2 + diff "$TARGET_FILE" - <<<"$GENERATED" + exit 1 + fi + ) + done +} + +usage() { + cat >&2 <<-EOF +Usage: $0 MODE + +Modes: + check - run the tests, compare the generated outputs with snapshots and + exit with non-zero exit code if the outputs do not match + update - run the tests and update the snapshots from the generated output +EOF + exit 1 +} + +MODE="${1:-}" +case "$MODE" in +update | check) + "$MODE" + ;; +*) + usage + ;; +esac diff --git a/tests/helm-snapshots/README.md b/tests/helm-snapshots/README.md new file mode 100644 index 0000000000000..72258056a251d --- /dev/null +++ b/tests/helm-snapshots/README.md @@ -0,0 +1,5 @@ +# Snapshot Tests for Helm Charts + +This directory contains snapshot tests for Helm charts. + +Snapshot tests are yet another layer of QA that we apply to Helm charts. diff --git a/tests/helm-snapshots/builtin_configs/vector-agent/config.sh b/tests/helm-snapshots/builtin_configs/vector-agent/config.sh new file mode 100644 index 0000000000000..26cef69b764c3 --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-agent/config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# shellcheck disable=SC2034 + +CHART="distribution/helm/vector-agent" +RELEASE_NAME="vector" diff --git a/tests/helm-snapshots/builtin_configs/vector-agent/snapshot.yaml b/tests/helm-snapshots/builtin_configs/vector-agent/snapshot.yaml new file mode 100644 index 0000000000000..e7c3f61563b8a --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-agent/snapshot.yaml @@ -0,0 +1,240 @@ +# Do not edit! +# This file is generated +# - by "scripts/helm-snapshot-tests.sh" +# - for the chart at "distribution/helm/vector-agent" +# - with the values from "tests/helm-snapshots/builtin_configs/vector-agent/values.yaml" +--- +# Source: vector-agent/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: vector-agent/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +data: + # We leave `vector.toml` file name available to let externally managed config + # maps to provide it. + managed.toml: | + # Configuration for vector. + # Docs: https://vector.dev/docs/ + + data_dir = "/vector-data-dir" + + [log_schema] + host_key = "host" + message_key = "message" + source_type_key = "source_type" + timestamp_key = "timestamp" + + # Ingest logs from Kubernetes. + [sources.kubernetes_logs] + option1 = "value1" + option2 = "value2" + type = "kubernetes_logs" + arbitrary text 1 + + # Send logs to the aggregator. + [sinks.vector_sink] + address = "vector.internal:9000" + inputs = ["my_input_1", "my_input_2"] + option1 = "value1" + option2 = "value2" + type = "vector" + arbitrary text 1 + + # Capture the metrics from the host. + [sources.host_metrics] + option1 = "value1" + option2 = "value2" + type = "host_metrics" + arbitrary text 1 + + + # Emit internal Vector metrics. + [sources.internal_metrics] + option1 = "value1" + option2 = "value2" + type = "internal_metrics" + arbitrary text 1 + + # Expose metrics for scraping in the Prometheus format. + [sinks.prometheus_sink] + address = "0.0.0.0:9090" + inputs = ["internal_metrics", "host_metrics"] + option1 = "value1" + option2 = "value2" + type = "prometheus" + arbitrary text 1 +--- +# Source: vector-agent/templates/rbac.yaml +# Permissions to use Kubernetes API. +# Requires that RBAC authorization is enabled. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vector-agent +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - watch +--- +# Source: vector-agent/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vector-agent +subjects: + - kind: ServiceAccount + name: vector-agent + namespace: vector +--- +# Source: vector-agent/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + minReadySeconds: 1 + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + + checksum/config: 75958f6034a2d836d57468a572df3af05ea5d4eaf4b8b5cbd6371e6ec319a696 + + labels: + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + vector.dev/exclude: "true" + spec: + serviceAccountName: vector-agent + securityContext: + {} + containers: + - name: vector + securityContext: + {} + image: "timberio/vector:0.0.0-debian" + imagePullPolicy: "IfNotPresent" + args: + - --config + - /etc/vector/*.toml + env: + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PROCFS_ROOT + value: /host/proc + - name: SYSFS_ROOT + value: /host/sys + + ports: + - name: metrics + containerPort: 9090 + protocol: TCP + resources: + {} + volumeMounts: + # Host log directory mount. + - name: var-log + mountPath: /var/log/ + readOnly: true + # Host mount for docker and containerd log file symlinks. + - name: var-lib + mountPath: /var/lib + readOnly: true + # Vector data dir mount. + - name: data-dir + mountPath: "/vector-data-dir" + # Vector config dir mount. + - name: config-dir + mountPath: /etc/vector + readOnly: true + # Host procsfs mount. + - name: procfs + mountPath: /host/proc + readOnly: true + # Host sysfs mount. + - name: sysfs + mountPath: /host/sys + readOnly: true + terminationGracePeriodSeconds: 60 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + # Log directory. + - name: var-log + hostPath: + path: /var/log/ + # Docker and containerd log files in Kubernetes are symlinks to this folder. + - name: var-lib + hostPath: + path: /var/lib/ + # Vector will store it's data here. + - name: data-dir + hostPath: + path: /var/lib/vector/ + # Vector config dir. + - name: config-dir + projected: + sources: + - configMap: + name: vector-agent + # Host procsfs. + - name: procfs + hostPath: + path: /proc + # Host sysfs. + - name: sysfs + hostPath: + path: /sys diff --git a/tests/helm-snapshots/builtin_configs/vector-agent/values.yaml b/tests/helm-snapshots/builtin_configs/vector-agent/values.yaml new file mode 100644 index 0000000000000..7a2f137eaeca0 --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-agent/values.yaml @@ -0,0 +1,38 @@ +kubernetesLogsSource: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +vectorSink: + enabled: true + inputs: ["my_input_1", "my_input_2"] + host: "vector.internal" + port: 9000 + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +internalMetricsSource: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +hostMetricsSource: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +prometheusSink: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 diff --git a/tests/helm-snapshots/builtin_configs/vector-aggregator/config.sh b/tests/helm-snapshots/builtin_configs/vector-aggregator/config.sh new file mode 100644 index 0000000000000..69f662476169c --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-aggregator/config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# shellcheck disable=SC2034 + +CHART="distribution/helm/vector-aggregator" +RELEASE_NAME="vector" diff --git a/tests/helm-snapshots/builtin_configs/vector-aggregator/snapshot.yaml b/tests/helm-snapshots/builtin_configs/vector-aggregator/snapshot.yaml new file mode 100644 index 0000000000000..637356b97cd62 --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-aggregator/snapshot.yaml @@ -0,0 +1,189 @@ +# Do not edit! +# This file is generated +# - by "scripts/helm-snapshot-tests.sh" +# - for the chart at "distribution/helm/vector-aggregator" +# - with the values from "tests/helm-snapshots/builtin_configs/vector-aggregator/values.yaml" +--- +# Source: vector-aggregator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: vector-aggregator/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +data: + # We leave `vector.toml` file name available to let externally managed config + # maps to provide it. + managed.toml: | + # Configuration for vector. + # Docs: https://vector.dev/docs/ + + data_dir = "/vector-data-dir" + + [log_schema] + host_key = "host" + message_key = "message" + source_type_key = "source_type" + timestamp_key = "timestamp" + + # Accept logs from Vector agents. + [sources.vector] + address = "0.0.0.0:9000" + option1 = "value1" + option2 = "value2" + type = "vector" + arbitrary text 1 + + + # Emit internal Vector metrics. + [sources.internal_metrics] + option1 = "value1" + option2 = "value2" + type = "internal_metrics" + arbitrary text 1 + + # Expose metrics for scraping in the Prometheus format. + [sinks.prometheus_sink] + address = "0.0.0.0:9090" + inputs = ["internal_metrics"] + option1 = "value1" + option2 = "value2" + type = "prometheus" + arbitrary text 1 +--- +# Source: vector-aggregator/templates/service-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: vector-aggregator-headless + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + clusterIP: None + ports: + + - name: vector + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector +--- +# Source: vector-aggregator/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + ports: + + - name: vector + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector +--- +# Source: vector-aggregator/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + serviceName: vector-aggregator-headless + selector: + matchLabels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + podManagementPolicy: "Parallel" + replicas: 1 + template: + metadata: + annotations: + + checksum/config: 2e32b2eca9854ebc0d3877ff9cb719ac16698a6e976a05151945729a41eb058d + + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + vector.dev/exclude: "true" + spec: + serviceAccountName: vector-aggregator + securityContext: + {} + containers: + - name: vector + securityContext: + {} + image: "timberio/vector:0.0.0-debian" + imagePullPolicy: "IfNotPresent" + args: + - --config + - /etc/vector/*.toml + env: + + ports: + - name: metrics + containerPort: 9090 + protocol: TCP + resources: + {} + volumeMounts: + # Vector data dir mount. + - name: data-dir + mountPath: "/vector-data-dir" + # Vector config dir mount. + - name: config-dir + mountPath: /etc/vector + readOnly: true + # Extra volumes. + terminationGracePeriodSeconds: 60 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + # Vector will store it's data here. + - name: data-dir + emptyDir: {} + # Vector config dir. + - name: config-dir + projected: + sources: + - configMap: + name: vector-aggregator + optional: true + volumeClaimTemplates: diff --git a/tests/helm-snapshots/builtin_configs/vector-aggregator/values.yaml b/tests/helm-snapshots/builtin_configs/vector-aggregator/values.yaml new file mode 100644 index 0000000000000..d14888ab1153f --- /dev/null +++ b/tests/helm-snapshots/builtin_configs/vector-aggregator/values.yaml @@ -0,0 +1,20 @@ +vectorSource: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +internalMetricsSource: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 + +prometheusSink: + config: + option1: value1 + option2: value2 + rawConfig: |- + arbitrary text 1 diff --git a/tests/helm-snapshots/topology_config/vector-agent/config.sh b/tests/helm-snapshots/topology_config/vector-agent/config.sh new file mode 100644 index 0000000000000..26cef69b764c3 --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-agent/config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# shellcheck disable=SC2034 + +CHART="distribution/helm/vector-agent" +RELEASE_NAME="vector" diff --git a/tests/helm-snapshots/topology_config/vector-agent/snapshot.yaml b/tests/helm-snapshots/topology_config/vector-agent/snapshot.yaml new file mode 100644 index 0000000000000..21b2dd7c94400 --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-agent/snapshot.yaml @@ -0,0 +1,280 @@ +# Do not edit! +# This file is generated +# - by "scripts/helm-snapshot-tests.sh" +# - for the chart at "distribution/helm/vector-agent" +# - with the values from "tests/helm-snapshots/topology_config/vector-agent/values.yaml" +--- +# Source: vector-agent/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: vector-agent/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +data: + # We leave `vector.toml` file name available to let externally managed config + # maps to provide it. + managed.toml: | + # Configuration for vector. + # Docs: https://vector.dev/docs/ + + data_dir = "/vector-data-dir" + + [log_schema] + host_key = "host" + message_key = "message" + source_type_key = "source_type" + timestamp_key = "timestamp" + + # Ingest logs from Kubernetes. + [sources.kubernetes_logs] + type = "kubernetes_logs" + + # Capture the metrics from the host. + [sources.host_metrics] + type = "host_metrics" + + + # Emit internal Vector metrics. + [sources.internal_metrics] + type = "internal_metrics" + + # Expose metrics for scraping in the Prometheus format. + [sinks.prometheus_sink] + address = "0.0.0.0:9090" + inputs = ["internal_metrics", "host_metrics"] + type = "prometheus" + + + [sources.source1] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [sources.source2] + optionA = "valueA" + type = "type2" + [sources.source2.optionB] + suboption = "valueB" + [sources.source2] + arbitrary text 2 + + [sources.source3] + type = "type3" + + [transforms.transform1] + inputs = ["input1", "input2"] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [transforms.transform2] + inputs = ["input2", "input1"] + optionA = "valueA" + type = "type2" + [transforms.transform2.optionB] + suboption = "valueB" + [transforms.transform2] + arbitrary text 2 + + [transforms.transform3] + inputs = [] + type = "type3" + + [sinks.sink1] + inputs = ["input1", "input2"] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [sinks.sink2] + inputs = ["input2", "input1"] + optionA = "valueA" + type = "type2" + [sinks.sink2.optionB] + suboption = "valueB" + [sinks.sink2] + arbitrary text 2 + + [sinks.sink3] + inputs = [] + type = "type3" +--- +# Source: vector-agent/templates/rbac.yaml +# Permissions to use Kubernetes API. +# Requires that RBAC authorization is enabled. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vector-agent +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - watch +--- +# Source: vector-agent/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vector-agent +subjects: + - kind: ServiceAccount + name: vector-agent + namespace: vector +--- +# Source: vector-agent/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vector-agent + labels: + helm.sh/chart: vector-agent-0.0.0 + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + minReadySeconds: 1 + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + + checksum/config: 75958f6034a2d836d57468a572df3af05ea5d4eaf4b8b5cbd6371e6ec319a696 + + labels: + app.kubernetes.io/name: vector-agent + app.kubernetes.io/instance: vector + vector.dev/exclude: "true" + spec: + serviceAccountName: vector-agent + securityContext: + {} + containers: + - name: vector + securityContext: + {} + image: "timberio/vector:0.0.0-debian" + imagePullPolicy: "IfNotPresent" + args: + - --config + - /etc/vector/*.toml + env: + - name: VECTOR_SELF_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: VECTOR_SELF_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: VECTOR_SELF_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PROCFS_ROOT + value: /host/proc + - name: SYSFS_ROOT + value: /host/sys + + ports: + - name: metrics + containerPort: 9090 + protocol: TCP + resources: + {} + volumeMounts: + # Host log directory mount. + - name: var-log + mountPath: /var/log/ + readOnly: true + # Host mount for docker and containerd log file symlinks. + - name: var-lib + mountPath: /var/lib + readOnly: true + # Vector data dir mount. + - name: data-dir + mountPath: "/vector-data-dir" + # Vector config dir mount. + - name: config-dir + mountPath: /etc/vector + readOnly: true + # Host procsfs mount. + - name: procfs + mountPath: /host/proc + readOnly: true + # Host sysfs mount. + - name: sysfs + mountPath: /host/sys + readOnly: true + terminationGracePeriodSeconds: 60 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + # Log directory. + - name: var-log + hostPath: + path: /var/log/ + # Docker and containerd log files in Kubernetes are symlinks to this folder. + - name: var-lib + hostPath: + path: /var/lib/ + # Vector will store it's data here. + - name: data-dir + hostPath: + path: /var/lib/vector/ + # Vector config dir. + - name: config-dir + projected: + sources: + - configMap: + name: vector-agent + # Host procsfs. + - name: procfs + hostPath: + path: /proc + # Host sysfs. + - name: sysfs + hostPath: + path: /sys diff --git a/tests/helm-snapshots/topology_config/vector-agent/values.yaml b/tests/helm-snapshots/topology_config/vector-agent/values.yaml new file mode 100644 index 0000000000000..b8ba126d2ef11 --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-agent/values.yaml @@ -0,0 +1,59 @@ +sources: + source1: + type: "type1" + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + source2: + type: "type2" + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + source3: + type: "type3" + +transforms: + transform1: + type: "type1" + inputs: ["input1", "input2"] + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + transform2: + type: "type2" + inputs: ["input2", "input1"] + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + transform3: + type: "type3" + inputs: [] + +sinks: + sink1: + type: "type1" + inputs: ["input1", "input2"] + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + sink2: + type: "type2" + inputs: ["input2", "input1"] + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + sink3: + type: "type3" + inputs: [] diff --git a/tests/helm-snapshots/topology_config/vector-aggregator/config.sh b/tests/helm-snapshots/topology_config/vector-aggregator/config.sh new file mode 100644 index 0000000000000..69f662476169c --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-aggregator/config.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# shellcheck disable=SC2034 + +CHART="distribution/helm/vector-aggregator" +RELEASE_NAME="vector" diff --git a/tests/helm-snapshots/topology_config/vector-aggregator/snapshot.yaml b/tests/helm-snapshots/topology_config/vector-aggregator/snapshot.yaml new file mode 100644 index 0000000000000..5b34308a73fbd --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-aggregator/snapshot.yaml @@ -0,0 +1,241 @@ +# Do not edit! +# This file is generated +# - by "scripts/helm-snapshot-tests.sh" +# - for the chart at "distribution/helm/vector-aggregator" +# - with the values from "tests/helm-snapshots/topology_config/vector-aggregator/values.yaml" +--- +# Source: vector-aggregator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +--- +# Source: vector-aggregator/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +data: + # We leave `vector.toml` file name available to let externally managed config + # maps to provide it. + managed.toml: | + # Configuration for vector. + # Docs: https://vector.dev/docs/ + + data_dir = "/vector-data-dir" + + [log_schema] + host_key = "host" + message_key = "message" + source_type_key = "source_type" + timestamp_key = "timestamp" + + # Accept logs from Vector agents. + [sources.vector] + address = "0.0.0.0:9000" + type = "vector" + + + # Emit internal Vector metrics. + [sources.internal_metrics] + type = "internal_metrics" + + # Expose metrics for scraping in the Prometheus format. + [sinks.prometheus_sink] + address = "0.0.0.0:9090" + inputs = ["internal_metrics"] + type = "prometheus" + + + [sources.source1] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [sources.source2] + optionA = "valueA" + type = "type2" + [sources.source2.optionB] + suboption = "valueB" + [sources.source2] + arbitrary text 2 + + [sources.source3] + type = "type3" + + [transforms.transform1] + inputs = ["input1", "input2"] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [transforms.transform2] + inputs = ["input2", "input1"] + optionA = "valueA" + type = "type2" + [transforms.transform2.optionB] + suboption = "valueB" + [transforms.transform2] + arbitrary text 2 + + [transforms.transform3] + inputs = [] + type = "type3" + + [sinks.sink1] + inputs = ["input1", "input2"] + option1 = "value1" + option2 = "value2" + type = "type1" + option = "value" + arbitrary text + + [sinks.sink2] + inputs = ["input2", "input1"] + optionA = "valueA" + type = "type2" + [sinks.sink2.optionB] + suboption = "valueB" + [sinks.sink2] + arbitrary text 2 + + [sinks.sink3] + inputs = [] + type = "type3" +--- +# Source: vector-aggregator/templates/service-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: vector-aggregator-headless + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + clusterIP: None + ports: + + - name: vector + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector +--- +# Source: vector-aggregator/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + ports: + + - name: vector + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector +--- +# Source: vector-aggregator/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: vector-aggregator + labels: + helm.sh/chart: vector-aggregator-0.0.0 + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + app.kubernetes.io/version: "0.0.0" + app.kubernetes.io/managed-by: Helm +spec: + serviceName: vector-aggregator-headless + selector: + matchLabels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + podManagementPolicy: "Parallel" + replicas: 1 + template: + metadata: + annotations: + + checksum/config: 2e32b2eca9854ebc0d3877ff9cb719ac16698a6e976a05151945729a41eb058d + + labels: + app.kubernetes.io/name: vector-aggregator + app.kubernetes.io/instance: vector + vector.dev/exclude: "true" + spec: + serviceAccountName: vector-aggregator + securityContext: + {} + containers: + - name: vector + securityContext: + {} + image: "timberio/vector:0.0.0-debian" + imagePullPolicy: "IfNotPresent" + args: + - --config + - /etc/vector/*.toml + env: + + ports: + - name: metrics + containerPort: 9090 + protocol: TCP + resources: + {} + volumeMounts: + # Vector data dir mount. + - name: data-dir + mountPath: "/vector-data-dir" + # Vector config dir mount. + - name: config-dir + mountPath: /etc/vector + readOnly: true + # Extra volumes. + terminationGracePeriodSeconds: 60 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + # Vector will store it's data here. + - name: data-dir + emptyDir: {} + # Vector config dir. + - name: config-dir + projected: + sources: + - configMap: + name: vector-aggregator + optional: true + volumeClaimTemplates: diff --git a/tests/helm-snapshots/topology_config/vector-aggregator/values.yaml b/tests/helm-snapshots/topology_config/vector-aggregator/values.yaml new file mode 100644 index 0000000000000..b8ba126d2ef11 --- /dev/null +++ b/tests/helm-snapshots/topology_config/vector-aggregator/values.yaml @@ -0,0 +1,59 @@ +sources: + source1: + type: "type1" + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + source2: + type: "type2" + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + source3: + type: "type3" + +transforms: + transform1: + type: "type1" + inputs: ["input1", "input2"] + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + transform2: + type: "type2" + inputs: ["input2", "input1"] + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + transform3: + type: "type3" + inputs: [] + +sinks: + sink1: + type: "type1" + inputs: ["input1", "input2"] + option1: "value1" + option2: "value2" + rawConfig: |- + option = "value" + arbitrary text + sink2: + type: "type2" + inputs: ["input2", "input1"] + optionA: "valueA" + optionB: + suboption: "valueB" + rawConfig: |- + arbitrary text 2 + sink3: + type: "type3" + inputs: [] From c750955256a7a07488b98df542730113c296f299 Mon Sep 17 00:00:00 2001 From: MOZGIII Date: Thu, 14 Jan 2021 22:44:35 +0300 Subject: [PATCH 016/769] fix(internal_metrics source): Rework defaults and a minimal scraping interval value (#5682) * fix(internal_metrics source): Rework defaults and set a minimal scraping interval Signed-off-by: MOZGIII * Switch back to u64 from f64 Signed-off-by: MOZGIII * Switch from error to a warning Signed-off-by: MOZGIII * Apply suggestions from code review Co-authored-by: Jesse Szwedko Signed-off-by: MOZGIII * Improve the internal assertion logic and warning Signed-off-by: MOZGIII Co-authored-by: Jesse Szwedko --- src/sources/internal_metrics.rs | 37 ++++++++++++--------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/sources/internal_metrics.rs b/src/sources/internal_metrics.rs index 39ef8fb4b66ea..1f3d6e0544d07 100644 --- a/src/sources/internal_metrics.rs +++ b/src/sources/internal_metrics.rs @@ -9,25 +9,14 @@ use futures::{stream, SinkExt, StreamExt}; use serde::{Deserialize, Serialize}; use tokio::time; -#[serde(deny_unknown_fields)] -#[derive(Deserialize, Serialize, Debug, Clone)] +#[derive(Deserialize, Serialize, Debug, Clone, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields, default)] pub struct InternalMetricsConfig { - #[serde(default = "default_scrape_interval_secs")] + #[derivative(Default(value = "2"))] scrape_interval_secs: u64, } -pub const fn default_scrape_interval_secs() -> u64 { - 2 -} - -impl Default for InternalMetricsConfig { - fn default() -> Self { - Self { - scrape_interval_secs: default_scrape_interval_secs(), - } - } -} - inventory::submit! { SourceDescription::new::("internal_metrics") } @@ -44,12 +33,13 @@ impl SourceConfig for InternalMetricsConfig { shutdown: ShutdownSignal, out: Pipeline, ) -> crate::Result { - Ok(Box::pin(run( - get_controller()?, - self.scrape_interval_secs, - out, - shutdown, - ))) + if self.scrape_interval_secs == 0 { + warn!( + "Interval set to 0 secs, this could result in high CPU utilization. It is suggested to use interval >= 1 secs.", + ); + } + let interval = time::Duration::from_secs(self.scrape_interval_secs); + Ok(Box::pin(run(get_controller()?, interval, out, shutdown))) } fn output_type(&self) -> DataType { @@ -63,15 +53,14 @@ impl SourceConfig for InternalMetricsConfig { async fn run( controller: &Controller, - interval: u64, + interval: time::Duration, out: Pipeline, shutdown: ShutdownSignal, ) -> Result<(), ()> { let mut out = out.sink_map_err(|error| error!(message = "Error sending internal metrics.", %error)); - let duration = time::Duration::from_secs(interval); - let mut interval = time::interval(duration).take_until(shutdown); + let mut interval = time::interval(interval).take_until(shutdown); while interval.next().await.is_some() { let metrics = capture_metrics(controller); out.send_all(&mut stream::iter(metrics).map(Ok)).await?; From 7fa399f6ebfac4885ca2158fe76e8b427e424d0d Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 14 Jan 2021 12:58:23 -0700 Subject: [PATCH 017/769] enhancement(remap): Open docs URL for help in VRL REPL (#5926) * Open docs URL for help command Signed-off-by: Luc Perkins * Change function name Signed-off-by: Luc Perkins * Use only one print statement Signed-off-by: Luc Perkins * Use println instead of eprintln Signed-off-by: Luc Perkins * Fix formatting issue Signed-off-by: Luc Perkins * Remove unnecessary static lifetime Signed-off-by: Luc Perkins * Use separate docs command instead of help Signed-off-by: Luc Perkins * Update init text to include new command Signed-off-by: Luc Perkins * Fix other merge conflict Signed-off-by: Luc Perkins * Open docs for specific functions Signed-off-by: Luc Perkins * Use 'help docs' instead of plain 'docs' Signed-off-by: Luc Perkins * Fix formatting Signed-off-by: Luc Perkins * Fix comment Signed-off-by: Luc Perkins * Use reference instead of cloning Signed-off-by: Luc Perkins * Fix formatting Signed-off-by: Luc Perkins * Adopt Clippy recommendation for handling Option Signed-off-by: Luc Perkins * Fix formatting issue introduced by Clippy Signed-off-by: Luc Perkins * Adopt new Clippy recommendation Signed-off-by: Luc Perkins * Update error message Signed-off-by: Luc Perkins * One last reformat Signed-off-by: Luc Perkins --- Cargo.lock | 13 ++++++++++ lib/remap-cli/Cargo.toml | 4 ++- lib/remap-cli/src/repl.rs | 53 ++++++++++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00beb07147784..55b88e65a7dda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5218,12 +5218,14 @@ version = "0.1.0" dependencies = [ "bytes 0.5.6", "exitcode", + "regex", "remap-functions", "remap-lang", "rustyline", "serde_json", "structopt 0.3.21", "thiserror", + "webbrowser", ] [[package]] @@ -7842,6 +7844,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecad156490d6b620308ed411cfee90d280b3cbd13e189ea0d3fada8acc89158a" +dependencies = [ + "web-sys", + "widestring", + "winapi 0.3.9", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/lib/remap-cli/Cargo.toml b/lib/remap-cli/Cargo.toml index f417ef6b60342..418e660000a75 100644 --- a/lib/remap-cli/Cargo.toml +++ b/lib/remap-cli/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] bytes = "0.5.6" +regex = { version = "1.3.9", default-features = false, optional = true } remap = { package = "remap-lang", path = "../remap-lang" } remap-functions = { path = "../remap-functions" } serde_json = "1" @@ -19,7 +20,8 @@ structopt = { version = "0.3", default-features = false } thiserror = "1" rustyline = { version = "7.0.0", default-features = false, optional = true } exitcode = "1.1.2" +webbrowser = { version = "0.5.5", default-features = false, optional = true } [features] default = ["repl"] -repl = ["rustyline"] +repl = ["regex", "rustyline", "webbrowser"] diff --git a/lib/remap-cli/src/repl.rs b/lib/remap-cli/src/repl.rs index 30c8c1b76d7bf..f4f518538aaab 100644 --- a/lib/remap-cli/src/repl.rs +++ b/lib/remap-cli/src/repl.rs @@ -1,5 +1,7 @@ use crate::Error; +use regex::Regex; use remap::{state, Object, Program, Runtime, Value}; +use remap_functions::all as funcs; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::{Highlighter, MatchingBracketHighlighter}; @@ -8,15 +10,20 @@ use rustyline::validate::{self, MatchingBracketValidator, ValidationResult, Vali use rustyline::{Context, Editor, Helper}; use std::borrow::Cow::{self, Borrowed, Owned}; +const DOCS_URL: &str = "https://vector.dev/docs/reference/remap"; + const HELP_TEXT: &str = " VRL REPL commands: - next Load the next object or create a new one - prev Load the previous object - exit Terminate the program + help docs Navigate to the VRL docs on the Vector website + help docs Navigate to the VRL docs for the specified function + next Load the next object or create a new one + prev Load the previous object + exit Terminate the program "; pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { let mut index = 0; + let func_docs_regex = Regex::new(r"^help\sdocs\s(\w{1,})$").unwrap(); let mut rt = Runtime::new(state::Program::default()); let mut rl = Editor::::new(); @@ -50,10 +57,12 @@ pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { > > To run the CLI in regular mode, add a program to your command. > -> Type `help` to learn more. -> `next` to either load the next object or create a new one. -> `prev` to load the previous object. -> `exit` to terminate the program. +> VRL REPL commands: +> help docs Navigate to the VRL docs on the Vector website +> help docs Navigate to the VRL docs for the specified function +> next Load the next object or create a new one +> prev Load the previous object +> exit Terminate the program > > Any other value is resolved to a TRL expression. > @@ -64,8 +73,10 @@ pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { let readline = rl.readline("$ "); match readline.as_deref() { Ok(line) if line == "help" => print_help_text(), - Ok(line) if line == "exit" => break, - Ok(line) if line == "quit" => break, + Ok(line) if line == "help docs" => open_url(DOCS_URL), + Ok(line) if line == "exit" || line == "quit" => break, + // Capture "help docs " + Ok(line) if func_docs_regex.is_match(line) => show_func_docs(line, &func_docs_regex), Ok(line) => { rl.add_history_entry(line); @@ -212,3 +223,27 @@ impl Validator for Repl { fn print_help_text() { println!("{}", HELP_TEXT); } + +fn open_url(url: &str) { + if let Err(err) = webbrowser::open(url) { + println!( + "couldn't open default web browser: {}\n\ + you can access the desired documentation at {}", + err, url + ); + } +} + +fn show_func_docs(line: &str, pattern: &Regex) { + // Unwrap is okay in both cases here, as there's guaranteed to be two matches ("help docs" and + // "help docs ") + let matches = pattern.captures(line).unwrap(); + let func_name = matches.get(1).unwrap().as_str(); + + if funcs().iter().any(|f| f.identifier() == func_name) { + let func_url = format!("{}/#{}", DOCS_URL, func_name); + open_url(&func_url); + } else { + println!("function name {} not recognized", func_name); + } +} From ecf0537e8fdf0a3d19fb4efc145dc431b3937355 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 14 Jan 2021 22:55:34 -0700 Subject: [PATCH 018/769] enhancement(remap): Pretty print functions list in VRL REPL (#5935) * Add functions helper to VRL CLI Signed-off-by: Luc Perkins * Allow for other keywords Signed-off-by: Luc Perkins * Change table format Signed-off-by: Luc Perkins * Fix formatting Signed-off-by: Luc Perkins * Fix formatting in help text Signed-off-by: Luc Perkins * Fix code formatting Signed-off-by: Luc Perkins * Make functions a sub-command of help Signed-off-by: Luc Perkins * s/functions/function Signed-off-by: Luc Perkins * Update help text Signed-off-by: Luc Perkins * s/function/functions Signed-off-by: Luc Perkins * s/TRL/VRL Signed-off-by: Luc Perkins * Update help text Signed-off-by: Luc Perkins * Fetch Cargo.lock from master Signed-off-by: Luc Perkins * Update Cargo.lock with new prettytable dependency Signed-off-by: Luc Perkins * Add comment on avoiding panic Signed-off-by: Luc Perkins * Remove help subcommands from initial message Signed-off-by: Luc Perkins --- Cargo.lock | 1 + lib/remap-cli/Cargo.toml | 3 ++- lib/remap-cli/src/repl.rs | 54 +++++++++++++++++++++++++++++++-------- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55b88e65a7dda..cb7452b5f7448 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5218,6 +5218,7 @@ version = "0.1.0" dependencies = [ "bytes 0.5.6", "exitcode", + "prettytable-rs", "regex", "remap-functions", "remap-lang", diff --git a/lib/remap-cli/Cargo.toml b/lib/remap-cli/Cargo.toml index 418e660000a75..22e5871f3406c 100644 --- a/lib/remap-cli/Cargo.toml +++ b/lib/remap-cli/Cargo.toml @@ -20,8 +20,9 @@ structopt = { version = "0.3", default-features = false } thiserror = "1" rustyline = { version = "7.0.0", default-features = false, optional = true } exitcode = "1.1.2" +prettytable-rs = { version = "0.8.0", default-features = false, optional = true } webbrowser = { version = "0.5.5", default-features = false, optional = true } [features] default = ["repl"] -repl = ["regex", "rustyline", "webbrowser"] +repl = ["prettytable-rs", "regex", "rustyline", "webbrowser"] diff --git a/lib/remap-cli/src/repl.rs b/lib/remap-cli/src/repl.rs index f4f518538aaab..2c571ec82bb7c 100644 --- a/lib/remap-cli/src/repl.rs +++ b/lib/remap-cli/src/repl.rs @@ -1,4 +1,5 @@ use crate::Error; +use prettytable::{format, Cell, Row, Table}; use regex::Regex; use remap::{state, Object, Program, Runtime, Value}; use remap_functions::all as funcs; @@ -10,16 +11,17 @@ use rustyline::validate::{self, MatchingBracketValidator, ValidationResult, Vali use rustyline::{Context, Editor, Helper}; use std::borrow::Cow::{self, Borrowed, Owned}; -const DOCS_URL: &str = "https://vector.dev/docs/reference/remap"; - -const HELP_TEXT: &str = " +const HELP_TEXT: &str = r#" VRL REPL commands: + help functions Display a list of currently available VRL functions (aliases: ["help funcs", "help fs"]) help docs Navigate to the VRL docs on the Vector website help docs Navigate to the VRL docs for the specified function next Load the next object or create a new one prev Load the previous object exit Terminate the program -"; +"#; + +const DOCS_URL: &str = "https://vector.dev/docs/reference/remap"; pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { let mut index = 0; @@ -30,7 +32,7 @@ pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { rl.set_helper(Some(Repl::new())); println!( - " + r#" > VVVVVVVV VVVVVVVVRRRRRRRRRRRRRRRRR LLLLLLLLLLL > V::::::V V::::::VR::::::::::::::::R L:::::::::L > V::::::V V::::::VR::::::RRRRRR:::::R L:::::::::L @@ -58,23 +60,26 @@ pub(crate) fn run(mut objects: Vec) -> Result<(), Error> { > To run the CLI in regular mode, add a program to your command. > > VRL REPL commands: -> help docs Navigate to the VRL docs on the Vector website -> help docs Navigate to the VRL docs for the specified function +> help Learn more about VRL > next Load the next object or create a new one > prev Load the previous object > exit Terminate the program > -> Any other value is resolved to a TRL expression. +> Any other value is resolved to a VRL expression. > -> Try it out now by typing `.` and hitting [enter] to see the result.\n" +> Try it out now by typing `.` and hitting [enter] to see the result. +"# ); loop { let readline = rl.readline("$ "); match readline.as_deref() { + Ok(line) if line == "exit" || line == "quit" => break, Ok(line) if line == "help" => print_help_text(), + Ok(line) if line == "help functions" || line == "help funcs" || line == "help fs" => { + print_function_list() + } Ok(line) if line == "help docs" => open_url(DOCS_URL), - Ok(line) if line == "exit" || line == "quit" => break, // Capture "help docs " Ok(line) if func_docs_regex.is_match(line) => show_func_docs(line, &func_docs_regex), Ok(line) => { @@ -220,6 +225,35 @@ impl Validator for Repl { } } +fn print_function_list() { + let table_format = *format::consts::FORMAT_NO_LINESEP_WITH_TITLE; + let all_funcs = remap_functions::all(); + + let num_columns = 3; + + let mut func_table = Table::new(); + func_table.set_format(table_format); + all_funcs + .chunks(num_columns) + .map(|funcs| { + // Because it's possible that some chunks are only partial, e.g. have only two Some(_) + // values when num_columns is 3. And so this logic below is necessary to avoid panics + // caused by inappropriately calling funcs.get(_) on a None. + let mut ids: Vec = Vec::new(); + + for n in 0..num_columns { + if let Some(v) = funcs.get(n) { + ids.push(Cell::new(v.identifier())); + } + } + + func_table.add_row(Row::new(ids)); + }) + .for_each(drop); + + func_table.printstd(); +} + fn print_help_text() { println!("{}", HELP_TEXT); } From 59b609073453f3cb50b1106186cee7bdf6534ef2 Mon Sep 17 00:00:00 2001 From: FungusHumungus Date: Fri, 15 Jan 2021 09:08:17 +0000 Subject: [PATCH 019/769] chore(remap): update functions that should be fallible (#6040) * Update functions that should be fallible Signed-off-by: Stephen Wakely * parse_grok is not fallible Signed-off-by: Stephen Wakely * Update docs with fallibility Signed-off-by: Stephen Wakely * Update ipv6_to_ipv4 docs to use the correct parameter name in the failure reason Signed-off-by: Stephen Wakely * Cue formatting Signed-off-by: Stephen Wakely * Formatting Signed-off-by: Stephen Wakely --- docs/reference/remap/assert.cue | 4 ++- docs/reference/remap/ipv6_to_ipv4.cue | 1 + lib/remap-functions/src/assert.rs | 7 +--- lib/remap-functions/src/ip_cidr_contains.rs | 9 ++--- lib/remap-functions/src/ip_subnet.rs | 9 ++--- lib/remap-functions/src/ip_to_ipv6.rs | 3 +- lib/remap-functions/src/ipv6_to_ipv4.rs | 3 +- lib/remap-functions/src/parse_aws_alb_log.rs | 1 - ...aws_cloudwatch_log_subscription_message.rs | 1 - .../src/parse_aws_vpc_flow_log.rs | 1 - lib/remap-functions/src/parse_duration.rs | 1 - lib/remap-functions/src/parse_json.rs | 1 - lib/remap-functions/src/parse_url.rs | 1 - lib/remap-functions/src/to_syslog_facility.rs | 33 ++++++----------- lib/remap-functions/src/to_syslog_level.rs | 35 ++++++------------- lib/remap-functions/src/to_syslog_severity.rs | 27 ++++++-------- 16 files changed, 47 insertions(+), 90 deletions(-) diff --git a/docs/reference/remap/assert.cue b/docs/reference/remap/assert.cue index f53250f5eaa2a..a5d06fd21ec07 100644 --- a/docs/reference/remap/assert.cue +++ b/docs/reference/remap/assert.cue @@ -15,7 +15,9 @@ remap: functions: assert: { type: ["string"] }, ] - internal_failure_reason: null + internal_failure_reason: [ + "`condition` evaluates to `false`", + ] return: ["null"] category: "Test" description: #""" diff --git a/docs/reference/remap/ipv6_to_ipv4.cue b/docs/reference/remap/ipv6_to_ipv4.cue index 63fd3a0067775..4b965069352f4 100644 --- a/docs/reference/remap/ipv6_to_ipv4.cue +++ b/docs/reference/remap/ipv6_to_ipv4.cue @@ -11,6 +11,7 @@ remap: functions: ipv6_to_ipv4: { ] internal_failure_reasons: [ "`ip` is not a valid IP address", + "`ip` is an IPv6 address that is not compatible with IPv4", ] return: ["string"] category: "IP" diff --git a/lib/remap-functions/src/assert.rs b/lib/remap-functions/src/assert.rs index fa1158a89a99b..712d1b56e7047 100644 --- a/lib/remap-functions/src/assert.rs +++ b/lib/remap-functions/src/assert.rs @@ -67,12 +67,7 @@ impl Expression for AssertFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.condition .type_def(state) - .fallible_unless(value::Kind::Boolean) - .merge_with_default_optional( - self.message - .as_ref() - .map(|message| message.type_def(state).fallible_unless(value::Kind::Bytes)), - ) + .into_fallible(true) .with_constraint(value::Kind::Null) } } diff --git a/lib/remap-functions/src/ip_cidr_contains.rs b/lib/remap-functions/src/ip_cidr_contains.rs index e3f1da44bb85b..94003d30775e3 100644 --- a/lib/remap-functions/src/ip_cidr_contains.rs +++ b/lib/remap-functions/src/ip_cidr_contains.rs @@ -66,12 +66,8 @@ impl Expression for IpCidrContainsFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) - .merge( - self.cidr - .type_def(state) - .fallible_unless(value::Kind::Bytes), - ) + .merge(self.cidr.type_def(state).into_fallible(true)) + .into_fallible(true) .with_constraint(value::Kind::Boolean) } } @@ -88,6 +84,7 @@ mod tests { }, def: TypeDef { kind: value::Kind::Boolean, + fallible: true, ..Default::default() }, }]; diff --git a/lib/remap-functions/src/ip_subnet.rs b/lib/remap-functions/src/ip_subnet.rs index 82fd938dd1496..92a852b2ff265 100644 --- a/lib/remap-functions/src/ip_subnet.rs +++ b/lib/remap-functions/src/ip_subnet.rs @@ -94,12 +94,8 @@ impl Expression for IpSubnetFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) - .merge( - self.subnet - .type_def(state) - .fallible_unless(value::Kind::Bytes), - ) + .merge(self.subnet.type_def(state).into_fallible(true)) + .into_fallible(true) .with_constraint(value::Kind::Bytes) } } @@ -168,6 +164,7 @@ mod tests { }, def: TypeDef { kind: value::Kind::Bytes, + fallible: true, ..Default::default() }, }]; diff --git a/lib/remap-functions/src/ip_to_ipv6.rs b/lib/remap-functions/src/ip_to_ipv6.rs index d91c43539909b..e1f6fa9f71586 100644 --- a/lib/remap-functions/src/ip_to_ipv6.rs +++ b/lib/remap-functions/src/ip_to_ipv6.rs @@ -55,7 +55,7 @@ impl Expression for IpToIpv6Fn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) + .into_fallible(true) .with_constraint(value::Kind::Bytes) } } @@ -71,6 +71,7 @@ mod tests { }, def: TypeDef { kind: value::Kind::Bytes, + fallible: true, ..Default::default() }, }]; diff --git a/lib/remap-functions/src/ipv6_to_ipv4.rs b/lib/remap-functions/src/ipv6_to_ipv4.rs index ccaa420adaadc..ddbfce2bf7249 100644 --- a/lib/remap-functions/src/ipv6_to_ipv4.rs +++ b/lib/remap-functions/src/ipv6_to_ipv4.rs @@ -58,7 +58,7 @@ impl Expression for Ipv6ToIpV4Fn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) + .into_fallible(true) .with_constraint(value::Kind::Bytes) } } @@ -74,6 +74,7 @@ mod tests { }, def: TypeDef { kind: value::Kind::Bytes, + fallible: true, ..Default::default() }, }]; diff --git a/lib/remap-functions/src/parse_aws_alb_log.rs b/lib/remap-functions/src/parse_aws_alb_log.rs index b66fbda199165..13c87f6d29467 100644 --- a/lib/remap-functions/src/parse_aws_alb_log.rs +++ b/lib/remap-functions/src/parse_aws_alb_log.rs @@ -53,7 +53,6 @@ impl Expression for ParseAwsAlbLogFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) .into_fallible(true) // Log parsing error .with_constraint(value::Kind::Map) } diff --git a/lib/remap-functions/src/parse_aws_cloudwatch_log_subscription_message.rs b/lib/remap-functions/src/parse_aws_cloudwatch_log_subscription_message.rs index ed21f6bee91aa..a38a93394ea63 100644 --- a/lib/remap-functions/src/parse_aws_cloudwatch_log_subscription_message.rs +++ b/lib/remap-functions/src/parse_aws_cloudwatch_log_subscription_message.rs @@ -57,7 +57,6 @@ impl Expression for ParseAwsCloudWatchLogSubscriptionMessageFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) .into_fallible(true) // Message parsing error .with_constraint(value::Kind::Map) } diff --git a/lib/remap-functions/src/parse_aws_vpc_flow_log.rs b/lib/remap-functions/src/parse_aws_vpc_flow_log.rs index 6bb60dce7f446..8cb487456d297 100644 --- a/lib/remap-functions/src/parse_aws_vpc_flow_log.rs +++ b/lib/remap-functions/src/parse_aws_vpc_flow_log.rs @@ -62,7 +62,6 @@ impl Expression for ParseAwsVpcFlowLogFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) .into_fallible(true) // Log parsing error .with_constraint(value::Kind::Map) } diff --git a/lib/remap-functions/src/parse_duration.rs b/lib/remap-functions/src/parse_duration.rs index eda7fa95c4f50..fce9e7888b6f0 100644 --- a/lib/remap-functions/src/parse_duration.rs +++ b/lib/remap-functions/src/parse_duration.rs @@ -119,7 +119,6 @@ impl Expression for ParseDurationFn { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) .merge(output_def) .into_fallible(true) // parsing errors .with_constraint(value::Kind::Float) diff --git a/lib/remap-functions/src/parse_json.rs b/lib/remap-functions/src/parse_json.rs index d93fdbb60a3fe..8ff43ea1fca5b 100644 --- a/lib/remap-functions/src/parse_json.rs +++ b/lib/remap-functions/src/parse_json.rs @@ -43,7 +43,6 @@ impl Expression for ParseJsonFn { self.value .type_def(state) - .fallible_unless(Kind::Bytes) .into_fallible(true) // JSON parsing errors .with_constraint( Kind::Bytes diff --git a/lib/remap-functions/src/parse_url.rs b/lib/remap-functions/src/parse_url.rs index 5431193ecdeec..f272adbf719c1 100644 --- a/lib/remap-functions/src/parse_url.rs +++ b/lib/remap-functions/src/parse_url.rs @@ -51,7 +51,6 @@ impl Expression for ParseUrlFn { fn type_def(&self, state: &state::Compiler) -> TypeDef { self.value .type_def(state) - .fallible_unless(value::Kind::Bytes) .into_fallible(true) // URL parsing error .with_constraint(value::Kind::Map) } diff --git a/lib/remap-functions/src/to_syslog_facility.rs b/lib/remap-functions/src/to_syslog_facility.rs index 861b03914a67c..015d1f60e1379 100644 --- a/lib/remap-functions/src/to_syslog_facility.rs +++ b/lib/remap-functions/src/to_syslog_facility.rs @@ -79,29 +79,16 @@ mod tests { use super::*; use value::Kind; - test_type_def![ - value_integer_non_fallible { - expr: |_| ToSyslogFacilityFn { - value: Literal::from(3).boxed(), - }, - def: TypeDef { - fallible: true, - kind: Kind::Bytes, - ..Default::default() - }, - } - - value_non_integer_fallible { - expr: |_| ToSyslogFacilityFn { - value: Literal::from("foo").boxed(), - }, - def: TypeDef { - fallible: true, - kind: Kind::Bytes, - ..Default::default() - }, - } - ]; + test_type_def![value_non_integer_fallible { + expr: |_| ToSyslogFacilityFn { + value: Literal::from("foo").boxed(), + }, + def: TypeDef { + fallible: true, + kind: Kind::Bytes, + ..Default::default() + }, + }]; test_function![ to_syslog_facility => ToSyslogFacility; diff --git a/lib/remap-functions/src/to_syslog_level.rs b/lib/remap-functions/src/to_syslog_level.rs index 25b79354fc198..3c944f3dee22e 100644 --- a/lib/remap-functions/src/to_syslog_level.rs +++ b/lib/remap-functions/src/to_syslog_level.rs @@ -56,7 +56,7 @@ impl Expression for ToSyslogLevelFn { self.value .type_def(state) - .fallible_unless(Kind::Integer) + .into_fallible(true) .with_constraint(Kind::Bytes) } } @@ -66,29 +66,16 @@ mod tests { use super::*; use value::Kind; - test_type_def![ - value_integer_non_fallible { - expr: |_| ToSyslogLevelFn { - value: Literal::from(3).boxed(), - }, - def: TypeDef { - fallible: false, - kind: Kind::Bytes, - ..Default::default() - }, - } - - value_non_integer_fallible { - expr: |_| ToSyslogLevelFn { - value: Literal::from("foo").boxed(), - }, - def: TypeDef { - fallible: true, - kind: Kind::Bytes, - ..Default::default() - }, - } - ]; + test_type_def![value_non_integer_fallible { + expr: |_| ToSyslogLevelFn { + value: Literal::from("foo").boxed(), + }, + def: TypeDef { + fallible: true, + kind: Kind::Bytes, + ..Default::default() + }, + }]; test_function![ to_syslog_level => ToSyslogLevel; diff --git a/lib/remap-functions/src/to_syslog_severity.rs b/lib/remap-functions/src/to_syslog_severity.rs index 78a16a0dd9e11..09f20a692a6ca 100644 --- a/lib/remap-functions/src/to_syslog_severity.rs +++ b/lib/remap-functions/src/to_syslog_severity.rs @@ -58,7 +58,7 @@ impl Expression for ToSyslogSeverityFn { self.value .type_def(state) - .fallible_unless(Kind::Bytes) + .into_fallible(true) .with_constraint(Kind::Integer) } } @@ -68,21 +68,16 @@ mod tests { use super::*; use value::Kind; - test_type_def![ - value_string_infallible { - expr: |_| ToSyslogSeverityFn { - value: Literal::from("warning").boxed(), - }, - def: TypeDef { fallible: false, kind: Kind::Integer, ..Default::default() }, - } - - value_not_string_fallible { - expr: |_| ToSyslogSeverityFn { - value: Literal::from(27).boxed(), - }, - def: TypeDef { fallible: true, kind: Kind::Integer, ..Default::default() }, - } - ]; + test_type_def![value_not_string_fallible { + expr: |_| ToSyslogSeverityFn { + value: Literal::from(27).boxed(), + }, + def: TypeDef { + fallible: true, + kind: Kind::Integer, + ..Default::default() + }, + }]; test_function![ to_level => ToSyslogSeverity; From 5e8f01dafceea6fce535d1ef4376eeb6406f33f2 Mon Sep 17 00:00:00 2001 From: FungusHumungus Date: Fri, 15 Jan 2021 14:36:15 +0000 Subject: [PATCH 020/769] chore(remap): remove only_fields remap function (#5861) * Remove only_fields Signed-off-by: Stephen Wakely * Remove object::paths Signed-off-by: Stephen Wakely * Move use FromStr to tests Signed-off-by: Stephen Wakely * Formatting Signed-off-by: Stephen Wakely --- lib/remap-functions/Cargo.toml | 2 - lib/remap-functions/src/lib.rs | 6 --- lib/remap-lang/src/object.rs | 5 --- lib/remap-lang/src/value/object.rs | 55 ------------------------ src/event/log_event.rs | 51 ----------------------- src/event/metric.rs | 62 ++-------------------------- tests/behavior/transforms/remap.toml | 29 ------------- 7 files changed, 3 insertions(+), 207 deletions(-) diff --git a/lib/remap-functions/Cargo.toml b/lib/remap-functions/Cargo.toml index e7410be3dc7c9..328fb7d1bd127 100644 --- a/lib/remap-functions/Cargo.toml +++ b/lib/remap-functions/Cargo.toml @@ -64,7 +64,6 @@ default = [ "md5", "merge", "now", - "only_fields", "parse_aws_alb_log", "parse_aws_cloudwatch_log_subscription_message", "parse_aws_vpc_flow_log", @@ -131,7 +130,6 @@ match = ["regex"] md5 = ["md-5", "hex"] merge = [] now = [] -only_fields = [] parse_aws_alb_log = ["nom"] parse_aws_cloudwatch_log_subscription_message = ["serde_json", "shared/aws_cloudwatch_logs_subscription"] parse_aws_vpc_flow_log = [] diff --git a/lib/remap-functions/src/lib.rs b/lib/remap-functions/src/lib.rs index e0260b30ff7ab..932eaf70a0451 100644 --- a/lib/remap-functions/src/lib.rs +++ b/lib/remap-functions/src/lib.rs @@ -54,8 +54,6 @@ mod md5; mod merge; #[cfg(feature = "now")] mod now; -#[cfg(feature = "only_fields")] -mod only_fields; #[cfg(feature = "parse_aws_alb_log")] mod parse_aws_alb_log; #[cfg(feature = "parse_aws_cloudwatch_log_subscription_message")] @@ -187,8 +185,6 @@ pub use log::Log; pub use merge::Merge; #[cfg(feature = "now")] pub use now::Now; -#[cfg(feature = "only_fields")] -pub use only_fields::OnlyFields; #[cfg(feature = "parse_aws_alb_log")] pub use parse_aws_alb_log::ParseAwsAlbLog; #[cfg(feature = "parse_aws_cloudwatch_log_subscription_message")] @@ -322,8 +318,6 @@ pub fn all() -> Vec> { Box::new(Merge), #[cfg(feature = "now")] Box::new(Now), - #[cfg(feature = "only_fields")] - Box::new(OnlyFields), #[cfg(feature = "parse_aws_alb_log")] Box::new(ParseAwsAlbLog), #[cfg(feature = "parse_aws_cloudwatch_log_subscription_message")] diff --git a/lib/remap-lang/src/object.rs b/lib/remap-lang/src/object.rs index 193bd1a5c8350..0ff26073ccfc6 100644 --- a/lib/remap-lang/src/object.rs +++ b/lib/remap-lang/src/object.rs @@ -46,11 +46,6 @@ pub trait Object: std::fmt::Debug { /// See [`Object::insert`] for more details. fn get(&self, path: &Path) -> Result, String>; - /// Get the list of paths in the object. - /// - /// Paths are represented similar to what's documented in [`Object::insert`]. - fn paths(&self) -> Result, String>; - /// Remove the given path from the object. /// /// If `compact` is true, after deletion, if an empty object or array is diff --git a/lib/remap-lang/src/value/object.rs b/lib/remap-lang/src/value/object.rs index dea8e91ff44f1..74dcd993b5348 100644 --- a/lib/remap-lang/src/value/object.rs +++ b/lib/remap-lang/src/value/object.rs @@ -10,10 +10,6 @@ impl Object for Value { Ok(self.get_by_path(path).cloned()) } - fn paths(&self) -> Result, String> { - self.paths().map_err(|err| err.to_string()) - } - fn remove(&mut self, path: &Path, compact: bool) -> Result, String> { let value = self.get(path)?; self.remove_by_path(path, compact); @@ -26,7 +22,6 @@ impl Object for Value { mod tests { use super::*; use crate::{value, Field::*, Segment::*}; - use std::str::FromStr; #[test] fn object_get() { @@ -278,54 +273,4 @@ mod tests { assert_eq!(Object::get(&object, &Path::root()), Ok(expect)); } } - - #[test] - fn object_paths() { - let cases = vec![ - (value!({}), Ok(vec![". "])), - ( - value!({"foo bar baz": "bar"}), - Ok(vec![". ", r#"."foo bar baz""#]), - ), - ( - value!({foo: "bar", baz: "qux"}), - Ok(vec![". ", ".baz", ".foo"]), - ), - ( - value!({foo: {bar: "baz"}}), - Ok(vec![". ", ".foo", ".foo.bar"]), - ), - (value!({a: [0, 1]}), Ok(vec![". ", ".a", ".a[0]", ".a[1]"])), - ( - value!({a: {b: "c"}, d: 12, e: [{f: 1}, {g: 2}, {h: 3}]}), - Ok(vec![ - ". ", ".a", ".a.b", ".d", ".e", ".e[0]", ".e[0].f", ".e[1]", ".e[1].g", - ".e[2]", ".e[2].h", - ]), - ), - ( - value!({a: [{b: [{c: {d: {e: [[0, 1]]}}}]}]}), - Ok(vec![ - ". ", - ".a", - ".a[0]", - ".a[0].b", - ".a[0].b[0]", - ".a[0].b[0].c", - ".a[0].b[0].c.d", - ".a[0].b[0].c.d.e", - ".a[0].b[0].c.d.e[0]", - ".a[0].b[0].c.d.e[0][0]", - ".a[0].b[0].c.d.e[0][1]", - ]), - ), - ]; - - for (object, expect) in cases { - assert_eq!( - Object::paths(&object), - expect.map(|vec| vec.iter().map(|s| Path::from_str(s).unwrap()).collect()) - ); - } - } } diff --git a/src/event/log_event.rs b/src/event/log_event.rs index 2e2b0fd509bfc..ea9bb7dacbde5 100644 --- a/src/event/log_event.rs +++ b/src/event/log_event.rs @@ -311,16 +311,6 @@ impl Object for LogEvent { Ok(()) } - - fn paths(&self) -> Result, String> { - if self.is_empty() { - return Ok(vec![remap::Path::root()]); - } - - self.keys() - .map(|key| remap::Path::from_alternative_string(key).map_err(|err| err.to_string())) - .collect() - } } #[cfg(test)] @@ -665,45 +655,4 @@ mod test { assert_eq!(Object::get(&event, &Path::root()), Ok(expect)) } } - - #[test] - fn object_paths() { - use crate::map; - use remap::{Object, Path}; - use std::str::FromStr; - - let cases = vec![ - (map![], Ok(vec!["."])), - (map!["foo bar baz": "bar"], Ok(vec![r#"."foo bar baz""#])), - (map!["foo": "bar", "baz": "qux"], Ok(vec![".baz", ".foo"])), - (map!["foo": map!["bar": "baz"]], Ok(vec![".foo.bar"])), - (map!["a": vec![0, 1]], Ok(vec![".a[0]", ".a[1]"])), - ( - map!["a": map!["b": "c"], "d": 12, "e": vec![ - map!["f": 1], - map!["g": 2], - map!["h": 3], - ]], - Ok(vec![".a.b", ".d", ".e[0].f", ".e[1].g", ".e[2].h"]), - ), - ( - map![ - "a": vec![map![ - "b": vec![map!["c": map!["d": map!["e": vec![vec![0, 1]]]]]] - ]] - ], - Ok(vec![".a[0].b[0].c.d.e[0][0]", ".a[0].b[0].c.d.e[0][1]"]), - ), - ]; - - for (object, expect) in cases { - let object: BTreeMap = object; - let event = LogEvent::from(object); - - assert_eq!( - event.paths(), - expect.map(|vec| vec.iter().map(|s| Path::from_str(s).unwrap()).collect()) - ); - } - } } diff --git a/src/event/metric.rs b/src/event/metric.rs index db3798f88fae2..9cf74e2b029fa 100644 --- a/src/event/metric.rs +++ b/src/event/metric.rs @@ -1,12 +1,9 @@ use chrono::{DateTime, Utc}; use derive_is_enum_variant::is_enum_variant; -use remap::{Object, Path, Segment}; +use remap::{Object, Segment}; use serde::{Deserialize, Serialize}; use snafu::Snafu; -use std::{ - collections::{BTreeMap, BTreeSet}, - str::FromStr, -}; +use std::collections::{BTreeMap, BTreeSet}; use std::{ convert::TryFrom, fmt::{self, Display, Formatter}, @@ -538,28 +535,6 @@ impl Object for Metric { } } - fn paths(&self) -> Result, String> { - let mut result = Vec::new(); - - result.push(Path::from_str("name").expect("invalid path")); - if self.namespace.is_some() { - result.push(Path::from_str("namespace").expect("invalid path")); - } - if self.timestamp.is_some() { - result.push(Path::from_str("timestamp").expect("invalid path")); - } - if let Some(tags) = &self.tags { - result.push(Path::from_str("tags").expect("invalid path")); - for name in tags.keys() { - result.push(Path::from_str(&format!("tags.{}", name)).expect("invalid path")); - } - } - result.push(Path::from_str("kind").expect("invalid path")); - result.push(Path::from_str("type").expect("invalid path")); - - Ok(result) - } - fn remove( &mut self, path: &remap::Path, @@ -625,6 +600,7 @@ mod test { use crate::map; use chrono::{offset::TimeZone, DateTime, Utc}; use remap::{Path, Value}; + use std::str::FromStr; fn ts() -> DateTime { Utc.ymd(2018, 11, 14).and_hms_nano(8, 9, 10, 11) @@ -966,38 +942,6 @@ mod test { ); } - #[test] - fn object_metric_paths() { - let metric = Metric { - name: "zub".into(), - namespace: Some("zoob".into()), - timestamp: Some(Utc.ymd(2020, 12, 10).and_hms(12, 0, 0)), - tags: Some({ - let mut map = BTreeMap::new(); - map.insert("tig".to_string(), "tog".to_string()); - map - }), - kind: MetricKind::Absolute, - value: MetricValue::Counter { value: 1.23 }, - }; - - assert_eq!( - Ok([ - "name", - "namespace", - "timestamp", - "tags", - "tags.tig", - "kind", - "type" - ] - .iter() - .map(|path| Path::from_str(path).expect("invalid path")) - .collect()), - metric.paths() - ); - } - #[test] fn object_metric_fields() { let mut metric = Metric { diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index fafb3790582d3..550be0804fac7 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -122,35 +122,6 @@ "result_c.equals" = true "result_d.equals" = false -[transforms.remap_delete_only_fields] - inputs = [] - type = "remap" - source = """ - only_fields(.foo, .bar, .buz.second) - del(.foo.second) - """ -[[tests]] - name = "remap_delete_only_fields" - [tests.input] - insert_at = "remap_delete_only_fields" - type = "log" - [tests.input.log_fields] - "foo.first" = "foo first value" - "foo.second" = "foo second value" - bar = "bar value" - baz = "baz value" - "buz.first" = "buz first value" - "buz.second" = "buz second value" - [[tests.outputs]] - extract_from = "remap_delete_only_fields" - [[tests.outputs.conditions]] - "foo.first.equals" = "foo first value" - "foo.second.exists" = false - "bar.equals" = "bar value" - "baz.exists" = false - "buz.first.exists" = false - "buz.second.equals" = "buz second value" - [transforms.remap_coercion] inputs = [] type = "remap" From b7e1234ab24aa33c155f093ed7a8e0f3a08c934b Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Fri, 15 Jan 2021 09:03:44 -0600 Subject: [PATCH 021/769] enhancement(prometheus_remote_write sink): Support X-Scope-OrgID header (#5987) * enhancement(prometheus_remote_write sink): Support X-Scope-OrgID header * Handle metrics in templates by treating them as empty logs Signed-off-by: Bruce Guenter --- .../sinks/prometheus_remote_write.cue | 11 + src/internal_events/prometheus.rs | 19 ++ src/sinks/prometheus/remote_write.rs | 210 +++++++++++++++--- src/template.rs | 11 +- 4 files changed, 210 insertions(+), 41 deletions(-) diff --git a/docs/reference/components/sinks/prometheus_remote_write.cue b/docs/reference/components/sinks/prometheus_remote_write.cue index 6986557826650..c8d92145e0e9d 100644 --- a/docs/reference/components/sinks/prometheus_remote_write.cue +++ b/docs/reference/components/sinks/prometheus_remote_write.cue @@ -126,6 +126,17 @@ components: sinks: prometheus_remote_write: { items: type: float: examples: [0.5, 0.75, 0.9, 0.95, 0.99] } } + tenant_id: { + common: false + description: "If set, a header named `X-Scope-OrgID` will be added to outgoing requests with the text of this setting. This may be used by Cortex or other remote services to identify the tenant making the request." + required: false + warnings: [] + type: string: { + default: null + examples: ["my-domain"] + templateable: true + } + } } input: { diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index a1c4d34ce4bcb..748853f0b8d60 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -158,3 +158,22 @@ impl InternalEvent for PrometheusServerRequestComplete { counter!("requests_received_total", 1); } } + +#[derive(Debug)] +pub struct PrometheusTemplateRenderingError { + pub fields: Vec, +} + +impl InternalEvent for PrometheusTemplateRenderingError { + fn emit_logs(&self) { + error!( + message = "Failed to render templated value; discarding value.", + fields = ?self.fields, + internal_log_rate_secs = 30, + ); + } + + fn emit_metrics(&self) { + counter!("processing_errors_total", 1); + } +} diff --git a/src/sinks/prometheus/remote_write.rs b/src/sinks/prometheus/remote_write.rs index 75e17dc659261..8b5dc39e174db 100644 --- a/src/sinks/prometheus/remote_write.rs +++ b/src/sinks/prometheus/remote_write.rs @@ -1,23 +1,27 @@ use super::collector::{self, MetricCollector as _}; use crate::{ config::{self, SinkConfig, SinkDescription}, - event::Metric, + event::{Event, Metric}, http::HttpClient, + internal_events::PrometheusTemplateRenderingError, sinks::{ self, util::{ - http::HttpRetryLogic, BatchConfig, BatchSettings, MetricBuffer, TowerRequestConfig, + http::HttpRetryLogic, BatchConfig, BatchSettings, MetricBuffer, PartitionBatchSink, + PartitionBuffer, PartitionInnerBuffer, TowerRequestConfig, }, }, + template::Template, tls::{TlsOptions, TlsSettings}, }; use bytes::{Bytes, BytesMut}; -use futures::{future::BoxFuture, FutureExt, SinkExt}; +use futures::{future::BoxFuture, stream, FutureExt, SinkExt}; use http::Uri; use prost::Message; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use std::task; +use tower::ServiceBuilder; #[derive(Debug, Snafu)] enum Errors { @@ -26,6 +30,7 @@ enum Errors { } #[derive(Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub(crate) struct RemoteWriteConfig { pub endpoint: String, @@ -41,6 +46,9 @@ pub(crate) struct RemoteWriteConfig { #[serde(default)] pub request: TowerRequestConfig, + #[serde(default)] + pub tenant_id: Option