Skip to content

Commit

Permalink
fix(codecs): Move protobuf codec options under a protobuf key (vect…
Browse files Browse the repository at this point in the history 
…ordotdev#18111)

Follow up to vectordotdev#18019 (comment)

The option "namespacing" is per: https://github.com/vectordotdev/vector/blob/master/docs/specs/configuration.md#polymorphism

Signed-off-by: Jesse Szwedko <[email protected]>
  • Loading branch information
@jszwedko
jszwedko authored Aug 1, 2023
1 parent 36111b5 commit 36788d1
Show file tree
Hide file tree
Showing 19 changed files with 327 additions and 223 deletions.
28 changes: 21 additions & 7 deletions lib/codecs/src/decoding/format/protobuf.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use std::path::PathBuf;

use bytes::Bytes;
use chrono::Utc;
use derivative::Derivative;
use ordered_float::NotNan;
use prost_reflect::{DescriptorPool, DynamicMessage, MessageDescriptor, ReflectMessage};
use smallvec::{smallvec, SmallVec};
Expand All @@ -22,11 +23,12 @@ use super::Deserializer;
#[configurable_component]
#[derive(Debug, Clone, Default)]
pub struct ProtobufDeserializerConfig {
/// Path to desc file
desc_file: PathBuf,

/// message type. e.g package.message
message_type: String,
/// Protobuf-specific decoding options.
#[serde(
default,
skip_serializing_if = "vector_core::serde::skip_serializing_if_default"
)]
pub protobuf: ProtobufDeserializerOptions,
}

impl ProtobufDeserializerConfig {
Expand Down Expand Up @@ -66,6 +68,18 @@ impl ProtobufDeserializerConfig {
}
}

/// Protobuf-specific decoding options.
#[configurable_component]
#[derive(Debug, Clone, PartialEq, Eq, Derivative)]
#[derivative(Default)]
pub struct ProtobufDeserializerOptions {
/// Path to desc file
desc_file: PathBuf,

/// message type. e.g package.message
message_type: String,
}

/// Deserializer that builds `Event`s from a byte frame containing protobuf.
#[derive(Debug, Clone)]
pub struct ProtobufDeserializer {
Expand Down Expand Up @@ -125,8 +139,8 @@ impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer {
type Error = vector_common::Error;
fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result<Self> {
let message_descriptor = ProtobufDeserializer::get_message_descriptor(
&config.desc_file,
config.message_type.clone(),
&config.protobuf.desc_file,
config.protobuf.message_type.clone(),
)?;
Ok(Self::new(message_descriptor))
}
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/amqp.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,6 @@ base: components: sources: amqp: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -138,12 +132,6 @@ base: components: sources: amqp: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -160,6 +148,23 @@ base: components: sources: amqp: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/aws_kinesis_firehose.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,6 @@ base: components: sources: aws_kinesis_firehose: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -141,12 +135,6 @@ base: components: sources: aws_kinesis_firehose: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -163,6 +151,23 @@ base: components: sources: aws_kinesis_firehose: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/aws_s3.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,6 @@ base: components: sources: aws_s3: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -236,12 +230,6 @@ base: components: sources: aws_s3: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -258,6 +246,23 @@ base: components: sources: aws_s3: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/aws_sqs.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,12 +193,6 @@ base: components: sources: aws_sqs: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -231,12 +225,6 @@ base: components: sources: aws_sqs: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -253,6 +241,23 @@ base: components: sources: aws_sqs: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/datadog_agent.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,6 @@ base: components: sources: datadog_agent: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -123,12 +117,6 @@ base: components: sources: datadog_agent: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -145,6 +133,23 @@ base: components: sources: datadog_agent: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
29 changes: 17 additions & 12 deletions website/cue/reference/components/sources/base/demo_logs.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,6 @@ base: components: sources: demo_logs: configuration: {
}
}
}
desc_file: {
description: "Path to desc file"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
gelf: {
description: "GELF-specific decoding options."
relevant_when: "codec = \"gelf\""
Expand Down Expand Up @@ -102,12 +96,6 @@ base: components: sources: demo_logs: configuration: {
type: bool: default: true
}
}
message_type: {
description: "message type. e.g package.message"
relevant_when: "codec = \"protobuf\""
required: true
type: string: {}
}
native_json: {
description: "Vector's native JSON-specific decoding options."
relevant_when: "codec = \"native_json\""
Expand All @@ -124,6 +112,23 @@ base: components: sources: demo_logs: configuration: {
type: bool: default: true
}
}
protobuf: {
description: "Protobuf-specific decoding options."
relevant_when: "codec = \"protobuf\""
required: false
type: object: options: {
desc_file: {
description: "Path to desc file"
required: false
type: string: default: ""
}
message_type: {
description: "message type. e.g package.message"
required: false
type: string: default: ""
}
}
}
syslog: {
description: "Syslog-specific decoding options."
relevant_when: "codec = \"syslog\""
Expand Down
Loading

0 comments on commit 36788d1

Please sign in to comment.