Merge remote-tracking branch 'origin' into stephen/event_cache_sink_t…

…ests Signed-off-by: Stephen Wakely <[email protected]>
vectordotdev · Jul 21, 2023 · 5f9c689 · 5f9c689
2 parents fa32c24 + 0bf6abd
commit 5f9c689
Show file tree

Hide file tree

Showing 243 changed files with 6,180 additions and 2,425 deletions.
diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md
@@ -41,8 +41,8 @@ On the day of release:
 - [ ] Merge release preparation branch into the release branch
     - `git co v0.<new version number> && git merge --ff-only prepare-v0.<new version number>`
 - [ ] Tag new release
-  - [ ] `git tag v0.<minor>.0 -a -m v0.<minor>.0``
-  - [ ] `git push origin v0.<minor>.0
+  - [ ] `git tag v0.<minor>.0 -a -m v0.<minor>.0`
+  - [ ] `git push origin v0.<minor>.0`
 - [ ] Wait for release workflow to complete
   - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml)
 - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing).

diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt
@@ -36,6 +36,7 @@ Celkon
 Ceph
 Chromecast
 Citrix
+Cloudflare
 Cloudfone
 Cmx
 Coby
@@ -165,6 +166,7 @@ Samsung
 Sega
 Segoe
 Shopify
+SIGINTs
 Simvalley
 Skype
 Skytex
@@ -244,6 +246,7 @@ compiletime
 coredns
 corejs
 coreutils
+curta
 daemonset
 databend
 datacenter
@@ -295,6 +298,9 @@ gpg
 gql
 grafana
 graphiql
+greptime
+greptimecloud
+greptimedb
 gvisor
 gws
 hadoop

diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -1,9 +1,7 @@
 abcd
 abcdefghijklm
 abcdefghijklmnopqrstuvwxyzand
-abced
 abortable
-acb
 ack'ing
 acking
 Acq
@@ -37,7 +35,6 @@ ARNOTAREALIDD
 arshiyasolei
 asdf
 asdfasdf
-ASMS
 assertverify
 Asterix
 asynk
@@ -140,7 +137,6 @@ casttype
 castvalue
 cbe
 CBOR
-cbs
 cddl
 cdeab
 cdylib
@@ -304,7 +300,6 @@ ebfcee
 edenhill
 edns
 eeyun
-efg
 efgh
 Elhage
 emca
@@ -325,7 +320,6 @@ ENVARS
 envsubst
 EOIG
 EOL'ed
-Erfxl
 Err'ing
 errorf
 Errorsfor
@@ -469,7 +463,6 @@ hannes
 Hashbang
 hashbrown
 hashindex
-hashlink
 hashring
 hashset
 hashsum
@@ -778,6 +771,7 @@ NQTP
 nresamples
 nullishness
 numbackends
+oahd
 oap
 OKD
 omfwd
@@ -821,6 +815,7 @@ pathgen
 peekable
 PEMS
 pgmajfault
+pgrep
 PII
 Pitbull
 pkc
@@ -1053,7 +1048,6 @@ supertrait
 suser
 sustainability
 svalue
-Sya
 sysfs
 sysinit
 syslogng
@@ -1114,7 +1108,6 @@ Tomola
 tonydanza
 toolbars
 toolchains
-TOOLSDIRECTORY
 toolset
 toor
 topdir

diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml
@@ -70,6 +70,8 @@ on:
         value: ${{ jobs.int_tests.outputs.fluent }}
       gcp:
         value: ${{ jobs.int_tests.outputs.gcp }}
+      greptimedb:
+        value: ${{ jobs.int_tests.outputs.greptimedb }}
       humio:
         value: ${{ jobs.int_tests.outputs.humio }}
       http-client:
@@ -194,6 +196,7 @@ jobs:
       eventstoredb: ${{ steps.filter.outputs.eventstoredb }}
       fluent: ${{ steps.filter.outputs.fluent }}
       gcp: ${{ steps.filter.outputs.gcp }}
+      greptimedb: ${{ steps.filter.outputs.greptimedb }}
       humio: ${{ steps.filter.outputs.humio }}
       http-client: ${{ steps.filter.outputs.http-client }}
       influxdb: ${{ steps.filter.outputs.influxdb }}

diff --git a/.github/workflows/comment-trigger.yml b/.github/workflows/comment-trigger.yml
@@ -15,6 +15,7 @@
 # /ci-run-unit-windows        : runs Unit - Windows
 # /ci-run-environment         : runs Environment Suite
 # /ci-run-regression          : runs Regression Detection Suite
+# /ci-run-k8s                 : runs K8s E2E Suite
 
 name: Comment Trigger
 
@@ -55,15 +56,22 @@ jobs:
           || contains(github.event.comment.body, '/ci-run-unit-windows')
           || contains(github.event.comment.body, '/ci-run-environment')
           || contains(github.event.comment.body, '/ci-run-regression')
+          || contains(github.event.comment.body, '/ci-run-k8s')
       )
     steps:
+      - name: Generate authentication token
+        id: generate_token
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92
+        with:
+          app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
+          private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
       - name: Get PR comment author
         id: comment
         uses: tspascoal/get-user-teams-membership@v2
         with:
           username: ${{ github.actor }}
           team: 'Vector'
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }}
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
 
       - name: Validate author membership
         if: steps.comment.outputs.isTeamMember == 'false'
@@ -116,3 +124,9 @@ jobs:
     if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-regression')
     uses: ./.github/workflows/regression.yml
     secrets: inherit
+
+  k8s:
+    needs: validate
+    if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-k8s')
+    uses: ./.github/workflows/k8s_e2e.yml
+    secrets: inherit
diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml
@@ -42,7 +42,7 @@ jobs:
       - name: Set up QEMU
         uses: docker/[email protected]
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.8.0
+        uses: docker/setup-buildx-action@v2.9.1
       - name: Login to DockerHub
         uses: docker/[email protected]
         if: github.ref == 'refs/heads/master'

diff --git a/.github/workflows/gardener_issue_comment.yml b/.github/workflows/gardener_issue_comment.yml
@@ -20,50 +20,74 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_PROJECT_PAT }}
         run: |
           issue_id=${{ github.event.issue.node_id }}
+          echo "issue_id: $issue_id"
 
           # IDs fetched from https://docs.github.com/en/graphql/overview/explorer
           project_id="PVT_kwDOAQFeYs4AAsTr" # Gardener
           status_field_id="PVTF_lADOAQFeYs4AAsTrzgAXRuU" # Status
           triage_option_id="2a08fafa"
 
-          # ensures that the issue is already on board but also seems to be the only way to fetch
-          # the item id
-          item_id="$(gh api graphql -f query='
-            mutation($project_id: ID!, $content_id: ID!) {
-                addProjectV2ItemById(input: {projectId: $project_id, contentId: $content_id}) {
-                  item {
-                    id
-                  }
-                }
-            }' -f project_id="$project_id" -f content_id="$issue_id" -q '.data.addProjectV2ItemById.item.id'
-          )"
-
-          echo "item_id: $item_id"
-
-          if [ -z "$item_id" ] ; then
-            echo "Issue not found in Gardener board"
-            exit 0
-          else
-            echo "Found issue on Gardener board"
-          fi
-
-          current_status="$(gh api graphql -f query='
+          # Query for project items for the given issue
+          project_items="$(gh api graphql -f query='
             query($item_id: ID!) {
               node(id: $item_id) {
-                ... on ProjectV2Item {
-                  fieldValueByName(name: "Status") {
-                    ... on ProjectV2ItemFieldSingleSelectValue {
-                      name
+                ... on Issue {
+                  projectItems(first: 50) {
+                    ... on ProjectV2ItemConnection {
+                      nodes {
+                        fieldValueByName(name: "Status") {
+                          ... on ProjectV2ItemFieldSingleSelectValue {
+                            name
+                          }
+                        }
+                        ... on ProjectV2Item {
+                          id
+                          project {
+                            ... on ProjectV2 {
+                              id
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+                ... on PullRequest {
+                  projectItems(first: 50) {
+                    ... on ProjectV2ItemConnection {
+                      nodes {
+                        fieldValueByName(name: "Status") {
+                          ... on ProjectV2ItemFieldSingleSelectValue {
+                            name
+                          }
+                        }
+                        ... on ProjectV2Item {
+                          id
+                          project {
+                            ... on ProjectV2 {
+                              id
+                            }
+                          }
+                        }
+                      }
                     }
                   }
                 }
               }
-            }' -f item_id="$item_id"
+            }' -f item_id="$issue_id"
           )"
 
-          current_status=$(echo $current_status | jq -c -r '.["data"]["node"]["fieldValueByName"]["name"]')
+          # Extract the item in the Gardener project
+          project=$(echo $project_items | jq -c -r --arg project_id $project_id '.data.node.projectItems.nodes[] | select(.project.id == $project_id)')
+          current_status=$(echo $project | jq -c '.fieldValueByName.name')
+          item_id=$(echo $project | jq -c '.id')
 
-          echo "Current issue status is: '${current_status}'"
+          if [ -z "$current_status" ] ; then
+            echo "Issue not found in Gardener board"
+            exit 0
+          else
+            echo "Found issue on Gardener board. Current issue status is: '${current_status}'"
+          fi
 
           if [ "$current_status" = "Blocked / Waiting" ] ; then
             echo "Moving issue from 'Blocked / Waiting' to 'Triage'"

diff --git a/.github/workflows/gardener_open_pr.yml b/.github/workflows/gardener_open_pr.yml
@@ -13,12 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
+      - name: Generate authentication token
+        id: generate_token
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92
+        with:
+          app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
+          private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
       - uses: tspascoal/get-user-teams-membership@v2
         id: checkVectorMember
         with:
           username: ${{ github.actor }}
           team: vector
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }}
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
       - uses: actions/[email protected]
         if: ${{ steps.checkVectorMember.outputs.isTeamMember == 'false' }}
         with: