Skip to content

Commit

Permalink
Merge branch 'master' into tobz/disk-buffer-event-metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
tobz committed Oct 10, 2023
2 parents 656bc5a + 4701bb9 commit 64d3f86
Show file tree
Hide file tree
Showing 41 changed files with 1,008 additions and 635 deletions.
52 changes: 26 additions & 26 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
docs/ @vectordotdev/ux-team
lib/file-source/ @spencergilbert @vectordotdev/integrations-team
lib/k8s-e2e-tests/ @spencergilbert @vectordotdev/integrations-team
lib/k8s-test-framework/ @spencergilbert @vectordotdev/integrations-team
lib/opentelemetry-proto/ @spencergilbert @vectordotdev/integrations-team
lib/file-source/ @vectordotdev/integrations-team
lib/k8s-e2e-tests/ @vectordotdev/integrations-team
lib/k8s-test-framework/ @vectordotdev/integrations-team
lib/opentelemetry-proto/ @vectordotdev/integrations-team
lib/vector-common/ @vectordotdev/core-team
lib/vector-config/ @vectordotdev/core-team
lib/vector-config-common/ @vectordotdev/core-team
Expand All @@ -15,39 +15,39 @@ src/internal_telemetry/ @vectordotdev/core-team
src/sinks/ @vectordotdev/integrations-team
src/sinks/amqp/ @StephenWakely @vectordotdev/integrations-team
src/sinks/appsignal/ @neuronull @vectordotdev/integrations-team
src/sinks/aws_cloudwatch_logs/ @spencergilbert @vectordotdev/integrations-team
src/sinks/aws_cloudwatch_metrics/ @spencergilbert @vectordotdev/integrations-team
src/sinks/aws_kinesis/ @spencergilbert @vectordotdev/integrations-team # sink_aws_kinesis_firehose,sink_aws_kinesis_stream
src/sinks/aws_s3/ @spencergilbert @vectordotdev/integrations-team
src/sinks/aws_sqs/ @spencergilbert @vectordotdev/integrations-team
src/sinks/axiom.rs @spencergilbert @vectordotdev/integrations-team
src/sinks/aws_cloudwatch_logs/ @vectordotdev/integrations-team
src/sinks/aws_cloudwatch_metrics/ @vectordotdev/integrations-team
src/sinks/aws_kinesis/ @vectordotdev/integrations-team # sink_aws_kinesis_firehose,sink_aws_kinesis_stream
src/sinks/aws_s3/ @vectordotdev/integrations-team
src/sinks/aws_sqs/ @vectordotdev/integrations-team
src/sinks/axiom.rs @vectordotdev/integrations-team
src/sinks/azure_blob/ @dsmith3197 @vectordotdev/integrations-team
src/sinks/azure_monitor_logs.rs @dsmith3197 @vectordotdev/integrations-team
src/sinks/blackhole/ @dsmith3197 @vectordotdev/integrations-team
src/sinks/clickhouse/ @dsmith3197 @vectordotdev/integrations-team
src/sinks/console/ @dsmith3197 @vectordotdev/integrations-team
src/sinks/databend/ @spencergilbert @vectordotdev/integrations-team
src/sinks/databend/ @vectordotdev/integrations-team
src/sinks/datadog_events/ @neuronull @vectordotdev/integrations-team
src/sinks/datadog_logs/ @neuronull @vectordotdev/integrations-team
src/sinks/datadog_metrics/ @neuronull @vectordotdev/integrations-team
src/sinks/datadog_traces/ @neuronull @vectordotdev/integrations-team
src/sinks/elasticsearch/ @spencergilbert @vectordotdev/integrations-team
src/sinks/file/ @spencergilbert @vectordotdev/integrations-team
src/sinks/elasticsearch/ @vectordotdev/integrations-team
src/sinks/file/ @vectordotdev/integrations-team
src/sinks/gcp/ @StephenWakely @vectordotdev/integrations-team # sink_gcp_chronicle_unstructured,sink_gcp_cloud_storage,sink_gcp_pubsub,sink_gcp_stackdriver_logs,sink_gcp_stackdriver_metrics
src/sinks/honeycomb.rs @spencergilbert @vectordotdev/integrations-team
src/sinks/honeycomb.rs @vectordotdev/integrations-team
src/sinks/http.rs @neuronull @vectordotdev/integrations-team
src/sinks/humio/ @StephenWakely @vectordotdev/integrations-team # sink_humio_logs,sink_humio_metrics
src/sinks/influxdb/ @dsmith3197 @vectordotdev/integrations-team # sink_influxdb_logs,sink_influxdb_metrics
src/sinks/kafka/ @dsmith3197 @vectordotdev/integrations-team
src/sinks/logdna.rs @neuronull @vectordotdev/integrations-team
src/sinks/loki/ @spencergilbert @vectordotdev/integrations-team
src/sinks/loki/ @vectordotdev/integrations-team
src/sinks/nats.rs @StephenWakely @vectordotdev/integrations-team
src/sinks/new_relic/ @dsmith3197 @vectordotdev/integrations-team # sink_newrelix,sink_newrelic_logs
src/sinks/papertrail.rs @StephenWakely @vectordotdev/integrations-team
src/sinks/prometheus/ @StephenWakely @vectordotdev/integrations-team # sink_prometheus_exporter,sink_prometheus_remote_write
src/sinks/pulsar.rs @dsmith3197 @vectordotdev/integrations-team
src/sinks/redis.rs @StephenWakely @vectordotdev/integrations-team
src/sinks/sematext/ @spencergilbert @vectordotdev/integrations-team # sink_sematext_logs,sink_sematext_metrics
src/sinks/sematext/ @vectordotdev/integrations-team # sink_sematext_logs,sink_sematext_metrics
src/sinks/socket.rs @neuronull @vectordotdev/integrations-team
src/sinks/splunk_hec/ @StephenWakely @vectordotdev/integrations-team # sink_splunk_hec_logs,sink_splunk_hec_metrics
src/sinks/statsd.rs @neuronull @vectordotdev/integrations-team
Expand All @@ -57,34 +57,34 @@ src/source_sender/ @vectordotdev/core-team
src/sources/ @vectordotdev/integrations-team
src/sources/amqp.rs @StephenWakely @vectordotdev/integrations-team
src/sources/apache_metrics/ @dsmith3197 @vectordotdev/integrations-team
src/sources/aws_ecs_metrics/ @spencergilbert @vectordotdev/integrations-team
src/sources/aws_kinesis_firehose/ @spencergilbert @vectordotdev/integrations-team
src/sources/aws_s3/ @spencergilbert @vectordotdev/integrations-team
src/sources/aws_sqs/ @spencergilbert @vectordotdev/integrations-team
src/sources/aws_ecs_metrics/ @vectordotdev/integrations-team
src/sources/aws_kinesis_firehose/ @vectordotdev/integrations-team
src/sources/aws_s3/ @vectordotdev/integrations-team
src/sources/aws_sqs/ @vectordotdev/integrations-team
src/sources/datadog_agent/ @neuronull @vectordotdev/integrations-team
src/sources/demo_logs.rs @StephenWakely @vectordotdev/integrations-team
src/sources/dnstap/ @StephenWakely @vectordotdev/integrations-team
src/sources/docker_logs/ @spencergilbert @vectordotdev/integrations-team
src/sources/docker_logs/ @vectordotdev/integrations-team
src/sources/eventstoredb_metrics/ @dsmith3197 @vectordotdev/integrations-team
src/sources/exec/ @dsmith3197 @vectordotdev/integrations-team
src/sources/file.rs @spencergilbert @vectordotdev/integrations-team
src/sources/file.rs @vectordotdev/integrations-team
src/sources/file_descriptors/ @dsmith3197 @vectordotdev/integrations-team # source_file_descriptor,source_stdin
src/sources/fluent/ @neuronull @vectordotdev/integrations-team
src/sources/gcp_pubsub.rs @StephenWakely @vectordotdev/integrations-team
src/sources/heroku_logs.rs @spencergilbert @vectordotdev/integrations-team
src/sources/heroku_logs.rs @vectordotdev/integrations-team
src/sources/host_metrics/ @dsmith3197 @vectordotdev/integrations-team
src/sources/http_client/ @neuronull @vectordotdev/integrations-team
src/sources/http_server.rs @neuronull @vectordotdev/integrations-team
src/sources/internal_logs.rs @neuronull @vectordotdev/integrations-team
src/sources/internal_metrics.rs @neuronull @vectordotdev/integrations-team
src/sources/journald.rs @spencergilbert @vectordotdev/integrations-team
src/sources/journald.rs @vectordotdev/integrations-team
src/sources/kafka.rs @dsmith3197 @vectordotdev/integrations-team
src/sources/kubernetes_logs/ @spencergilbert @vectordotdev/integrations-team
src/sources/kubernetes_logs/ @vectordotdev/integrations-team
src/sources/logstash.rs @neuronull @vectordotdev/integrations-team
src/sources/mongodb_metrics/ @dsmith3197 @vectordotdev/integrations-team
src/sources/nats.rs @StephenWakely @vectordotdev/integrations-team
src/sources/nginx_metrics/ @dsmith3197 @vectordotdev/integrations-team
src/sources/opentelemetry/ @spencergilbert @vectordotdev/integrations-team
src/sources/opentelemetry/ @vectordotdev/integrations-team
src/sources/postgresql_metrics.rs @dsmith3197 @vectordotdev/integrations-team
src/sources/prometheus/ @StephenWakely @vectordotdev/integrations-team # source_prometheus_remote_write,source_prometheus_scrape
src/sources/redis/ @StephenWakely @vectordotdev/integrations-team
Expand Down
3 changes: 3 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ updates:
clap:
patterns:
- "clap*"
csv:
patterns:
- "csv*"
phf:
patterns:
- "phf*"
Expand Down
18 changes: 12 additions & 6 deletions .github/workflows/changes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,14 @@ on:
value: ${{ jobs.int_tests.outputs.clickhouse }}
databend:
value: ${{ jobs.int_tests.outputs.databend }}
datadog:
value: ${{ jobs.int_tests.outputs.datadog }}
datadog-agent:
value: ${{ jobs.int_tests.outputs.datadog-agent }}
datadog-logs:
value: ${{ jobs.int_tests.outputs.datadog-logs }}
datadog-metrics:
value: ${{ jobs.int_tests.outputs.datadog-metrics }}
datadog-traces:
value: ${{ jobs.int_tests.outputs.datadog-traces }}
dnstap:
value: ${{ jobs.int_tests.outputs.dnstap }}
docker-logs:
Expand Down Expand Up @@ -143,9 +149,6 @@ jobs:
- "Makefile"
- "rust-toolchain.toml"
- "vdev/**"
deny:
- 'deny.toml'
- "vdev/**"
dependencies:
- ".cargo/**"
- 'Cargo.toml'
Expand Down Expand Up @@ -189,7 +192,10 @@ jobs:
azure: ${{ steps.filter.outputs.azure }}
clickhouse: ${{ steps.filter.outputs.clickhouse }}
databend: ${{ steps.filter.outputs.databend }}
datadog: ${{ steps.filter.outputs.datadog }}
datadog-agent: ${{ steps.filter.outputs.datadog-agent }}
datadog-logs: ${{ steps.filter.outputs.datadog-logs }}
datadog-metrics: ${{ steps.filter.outputs.datadog-metrics }}
datadog-traces: ${{ steps.filter.outputs.datadog-traces }}
dnstap: ${{ steps.filter.outputs.dnstap }}
docker-logs: ${{ steps.filter.outputs.docker-logs }}
elasticsearch: ${{ steps.filter.outputs.elasticsearch }}
Expand Down
10 changes: 9 additions & 1 deletion .github/workflows/comment-trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# /ci-run-all : runs all of the below
# /ci-run-cli : runs CLI - Linux
# /ci-run-misc : runs Miscellaneous - Linux
# /ci-run-deny : runs Deny - Linux
# /ci-run-component-features : runs Component Features - Linux
# /ci-run-cross : runs Cross
# /ci-run-unit-mac : runs Unit - Mac
Expand Down Expand Up @@ -50,6 +51,7 @@ jobs:
github.event.issue.pull_request && ( contains(github.event.comment.body, '/ci-run-all')
|| contains(github.event.comment.body, '/ci-run-cli')
|| contains(github.event.comment.body, '/ci-run-misc')
|| contains(github.event.comment.body, '/ci-run-deny')
|| contains(github.event.comment.body, '/ci-run-component-features')
|| contains(github.event.comment.body, '/ci-run-cross')
|| contains(github.event.comment.body, '/ci-run-unit-mac')
Expand All @@ -67,7 +69,7 @@ jobs:
private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
- name: Get PR comment author
id: comment
uses: tspascoal/get-user-teams-membership@v2
uses: tspascoal/get-user-teams-membership@v3
with:
username: ${{ github.actor }}
team: 'Vector'
Expand All @@ -89,6 +91,12 @@ jobs:
uses: ./.github/workflows/misc.yml
secrets: inherit

deny:
needs: validate
if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-deny')
uses: ./.github/workflows/deny.yml
secrets: inherit

component-features:
needs: validate
if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-component-features')
Expand Down
74 changes: 74 additions & 0 deletions .github/workflows/deny.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Deny - Linux
#
# Checks for security vulnerabilities or license incompatibilities
#
# Runs on:
# - scheduled UTC midnight
# - on PR comment (see comment-trigger.yml)
# - on demand from github actions UI

name: Deny - Linux

on:
workflow_call:
workflow_dispatch:
schedule:
# At midnight UTC
- cron: '0 0 * * *'

jobs:
test-deny:
runs-on: ubuntu-latest
env:
CARGO_INCREMENTAL: 0
steps:
- name: (PR comment) Get PR branch
if: ${{ github.event_name == 'issue_comment' }}
uses: xt0rted/pull-request-comment-branch@v2
id: comment-branch

- name: (PR comment) Set latest commit status as pending
if: ${{ github.event_name == 'issue_comment' }}
uses: myrotvorets/[email protected]
with:
sha: ${{ steps.comment-branch.outputs.head_sha }}
token: ${{ secrets.GITHUB_TOKEN }}
context: Deny - Linux
status: pending

- name: (PR comment) Checkout PR branch
if: ${{ github.event_name == 'issue_comment' }}
uses: actions/checkout@v3
with:
ref: ${{ steps.comment-branch.outputs.head_ref }}

- name: Checkout branch
if: ${{ github.event_name != 'issue_comment' }}
uses: actions/checkout@v3

- uses: actions/cache@v3
name: Cache Cargo registry + index
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-
- run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
- run: bash scripts/environment/prepare.sh
- run: echo "::add-matcher::.github/matchers/rust.json"
- name: Check cargo deny advisories/licenses
run: make check-deny

- name: (PR comment) Set latest commit status as ${{ job.status }}
uses: myrotvorets/[email protected]
if: always() && github.event_name == 'issue_comment'
with:
sha: ${{ steps.comment-branch.outputs.head_sha }}
token: ${{ secrets.GITHUB_TOKEN }}
context: Deny - Linux
status: ${{ job.status }}
2 changes: 1 addition & 1 deletion .github/workflows/gardener_open_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
with:
app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
- uses: tspascoal/get-user-teams-membership@v2
- uses: tspascoal/get-user-teams-membership@v3
id: checkVectorMember
with:
username: ${{ github.actor }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/integration-comment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
- name: Get PR comment author
id: comment
uses: tspascoal/get-user-teams-membership@v2
uses: tspascoal/get-user-teams-membership@v3
with:
username: ${{ github.actor }}
team: 'Vector'
Expand Down Expand Up @@ -387,7 +387,7 @@ jobs:
private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
- name: Validate issue comment
if: github.event_name == 'issue_comment'
uses: tspascoal/get-user-teams-membership@v2
uses: tspascoal/get-user-teams-membership@v3
with:
username: ${{ github.actor }}
team: 'Vector'
Expand Down
13 changes: 8 additions & 5 deletions .github/workflows/integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,10 @@ jobs:
|| needs.changes.outputs.azure == 'true'
|| needs.changes.outputs.clickhouse == 'true'
|| needs.changes.outputs.databend == 'true'
|| needs.changes.outputs.datadog == 'true'
|| needs.changes.outputs.datadog-agent == 'true'
|| needs.changes.outputs.datadog-logs == 'true'
|| needs.changes.outputs.datadog-metrics == 'true'
|| needs.changes.outputs.datadog-traces == 'true'
|| needs.changes.outputs.dnstap == 'true'
|| needs.changes.outputs.docker-logs == 'true'
|| needs.changes.outputs.elasticsearch == 'true'
Expand Down Expand Up @@ -166,7 +169,7 @@ jobs:
max_attempts: 3
command: bash scripts/ci-integration-test.sh databend

- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-agent
uses: nick-fields/retry@v2
Expand All @@ -175,7 +178,7 @@ jobs:
max_attempts: 3
command: bash scripts/ci-integration-test.sh datadog-agent

- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-logs
uses: nick-fields/retry@v2
Expand All @@ -184,7 +187,7 @@ jobs:
max_attempts: 3
command: bash scripts/ci-integration-test.sh datadog-logs

- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-metrics == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-metrics
uses: nick-fields/retry@v2
Expand All @@ -193,7 +196,7 @@ jobs:
max_attempts: 3
command: bash scripts/ci-integration-test.sh datadog-metrics

- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') &&
- if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') &&
(github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
name: datadog-traces
uses: nick-fields/retry@v2
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/protobuf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
# Run `git checkout`
- uses: actions/checkout@v3
# Install the `buf` CLI
- uses: bufbuild/buf-setup-action@v1.26.1
- uses: bufbuild/buf-setup-action@v1.27.0
# Perform breaking change detection against the `master` branch
- uses: bufbuild/[email protected]
with:
Expand Down
Loading

0 comments on commit 64d3f86

Please sign in to comment.