syncing up to e60876ef3f225d4f9f7433fec87eefbdb0b2f7f9

Co-authored-by: Joey Greco <[email protected]>

.yamlfmt

@@ -7,14 +7,15 @@ indent: 2
 disallow_anchors: false
 
 include:
-  - '.*\.(yaml|yml)$'
+- .*\.(yaml|yml)$
 
 exclude:
-  - helm/orchestrator/templates/
-  - helm/agent/templates/
-  - workers/golang/helm/templates/
-  - workers/python/helm/templates/
-  - workers/javascript/helm/templates/
-  - workers/javascript/pnpm-lock.yaml
-  - helm/cloud-opa/templates/
-  - clients/
+- .github/workflows/
+- helm/orchestrator/templates/
+- helm/agent/templates/
+- workers/golang/helm/templates/
+- workers/python/helm/templates/
+- workers/javascript/helm/templates/
+- workers/javascript/pnpm-lock.yaml
+- helm/cloud-opa/templates/
+- clients/

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added connection string support for RedShift, MySQL, MariaDB, Postgres, CockroachDB
 - Fix Snowflake template version
 - Fix connection for Snowflake integration so we do not hang for 90 seconds on an invalid configuration
+- Added AWS Kinesis plugin
 
 ## v1.14.1
 

compose.full.yaml

@@ -14,7 +14,7 @@ x-environment: &environment
   SUPERBLOCKS_AGENT_KEY: dev-agent-key
   SUPERBLOCKS_AGENT_REDIS_TOKEN: dev-agent-key
   SUPERBLOCKS_AGENT_REDIS_KVSTORE_TOKEN: dev-agent-key
-  SUPERBLOCKS_AGENT_TLS_INSECURE: "true"
+  SUPERBLOCKS_AGENT_TLS_INSECURE: 'true'
   SUPERBLOCKS_AGENT_REDIS_HOST: redis
   SUPERBLOCKS_AGENT_REDIS_KVSTORE_HOST: redis
   SUPERBLOCKS_AGENT_REDIS_KVSTORE_MAX_SIZE_BYTES: 10000000
@@ -25,24 +25,24 @@ x-environment-golang: &environment-golang
   SUPERBLOCKS_WORKER_GO_SUPERBLOCKS_KEY: dev-agent-key
   SUPERBLOCKS_WORKER_GO_STORE_REDIS_PASSWORD: dev-agent-key
   SUPERBLOCKS_WORKER_GO_STORE_REDIS_HOST: redis
-  SUPERBLOCKS_WORKER_GO_STORE_REDIS_TLS: "false"
+  SUPERBLOCKS_WORKER_GO_STORE_REDIS_TLS: 'false'
   SUPERBLOCKS_WORKER_GO_TRANSPORT_REDIS_PASSWORD: dev-agent-key
   SUPERBLOCKS_WORKER_GO_TRANSPORT_REDIS_HOST: redis
-  SUPERBLOCKS_WORKER_GO_TRANSPORT_REDIS_TLS: "false"
+  SUPERBLOCKS_WORKER_GO_TRANSPORT_REDIS_TLS: 'false'
 x-environment-lang: &environment-lang
-  SUPERBLOCKS_AGENT_PROCESS_ASYNC: "false"
+  SUPERBLOCKS_AGENT_PROCESS_ASYNC: 'false'
   SUPERBLOCKS_AGENT_PLUGIN_EVENTS: execute
 services:
   redis:
     image: ghcr.io/superblocksteam/redis:6
     ports:
-      - "6379:6379"
-    command: "--requirepass dev-agent-key"
+      - 6379:6379
+    command: --requirepass dev-agent-key
     pull_policy: always
   postgres:
     image: ghcr.io/superblocksteam/postgres:16.1
     ports:
-      - "15432:5432"
+      - 15432:5432
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=password
@@ -64,24 +64,34 @@ services:
       - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
       - --advertise-kafka-addr internal://kafka:9092,external://localhost:19092
     ports:
-      - "18081:8081"
-      - "18082:8082"
-      - "19092:19092"
-      - "9644:9644"
+      - 18081:8081
+      - 18082:8082
+      - 19092:19092
+      - 9644:9644
   kafka-setup:
     image: docker.redpanda.com/redpandadata/redpanda:latest
     depends_on:
       - kafka
-    restart: "no"
+    restart: no
     # TODO(dlamotte): may be able to avoid the timeout/while/sleep loop
     # by using compose health checks. I tried initially, but didn't work on
     # the first try.
-    entrypoint: ["bash", "-c", "if ! timeout 30 bash -c 'while ! rpk -X brokers=kafka:9092 topic list; do sleep 1; done'; then exit 1; fi; rpk -X brokers=kafka:9092 topic create metadata.cloud && rpk -X brokers=kafka:9092 topic create intake.metadata.dymamo && rpk -X brokers=kafka:9092 topic create intake.metadata.vector && rpk -X brokers=kafka:9092 topic create events"]
+    entrypoint: [bash, -c, 'if ! timeout 30 bash -c ''while ! rpk -X brokers=kafka:9092 topic list; do sleep 1; done''; then exit 1; fi; rpk -X brokers=kafka:9092 topic create metadata.cloud && rpk -X brokers=kafka:9092 topic create intake.metadata.dymamo && rpk -X brokers=kafka:9092 topic create intake.metadata.vector && rpk -X brokers=kafka:9092 topic create events']
+  localstack:
+    image: localstack/localstack:latest
+    environment:
+      - SERVICES=kinesis
+    ports:
+      - 4566:4566
+    # auto runs this setup script when ready
+    # https://stackoverflow.com/questions/53563942/running-shell-script-against-localstack-in-docker-container
+    volumes:
+      - ./scripts/plugins/integration/kinesis/init-kinesis.sh:/etc/localstack/init/ready.d/init-kinesis.sh
   intake:
     image: ghcr.io/superblocksteam/intake:v0.88.0
     ports:
-      - "3030:3030"
-      - "10300:10300"
+      - 3030:3030
+      - 10300:10300
     environment:
       SUPERBLOCKS_INTAKE_KAFKA_BOOTSTRAP: kafka:9092
       SUPERBLOCKS_INTAKE_DB_URL: postgres://postgres:password@postgres:5432/postgres?sslmode=disable
@@ -96,14 +106,14 @@ services:
       # from wherever orchestrator is running
       MOCK_OAUTH_SERVER_URL: http://${CI_ONLY_MOCK_OAUTH_SERVER-127.0.0.1}:5678
     ports:
-      - "3100:3100"
-      - "3110:3110"
+      - 3100:3100
+      - 3110:3110
   auxillary:
     !!merge <<: *build-js
     environment:
       !!merge <<: *environment
-      SUPERBLOCKS_AGENT_PROCESS_ASYNC: "true"
-      SUPERBLOCKS_AGENT_PLUGIN_EVENTS: "!execute"
+      SUPERBLOCKS_AGENT_PROCESS_ASYNC: 'true'
+      SUPERBLOCKS_AGENT_PLUGIN_EVENTS: '!execute'
       SUPERBLOCKS_AGENT_BUCKET: BA
       SUPERBLOCKS_AGENT_REDIS_GROUP: one
       FLEET: all
@@ -122,7 +132,7 @@ services:
     !!merge <<: [*build-js, *deploy]
     environment:
       !!merge <<: *environment
-      SUPERBLOCKS_AGENT_PROCESS_ASYNC: "true"
+      SUPERBLOCKS_AGENT_PROCESS_ASYNC: 'true'
       SUPERBLOCKS_AGENT_PLUGIN_EVENTS: execute
       SUPERBLOCKS_AGENT_BUCKET: BA
       SUPERBLOCKS_AGENT_REDIS_GROUP: three
@@ -131,27 +141,27 @@ services:
   # deploy javascript with 2 replicas for pool
   # tests
   javascript.b1.execute:
-    !!merge <<: [*build-js]
+    !!merge <<: *build-js
     deploy:
       replicas: 2
     environment:
-      !!merge <<: [*environment]
+      !!merge <<: *environment
       SUPERBLOCKS_AGENT_BUCKET: B1
       SUPERBLOCKS_AGENT_REDIS_GROUP: four
       FLEET: javascript
     depends_on: [redis]
   javascript.b2.execute:
-    !!merge <<: [*build-js]
+    !!merge <<: *build-js
     deploy:
       replicas: 2
     environment:
-      !!merge <<: [*environment]
+      !!merge <<: *environment
       SUPERBLOCKS_AGENT_BUCKET: B2
       SUPERBLOCKS_AGENT_REDIS_GROUP: five
       FLEET: javascript
     depends_on: [redis]
   javascript.ba.execute:
-    !!merge <<: [*build-js]
+    !!merge <<: *build-js
     deploy:
       replicas: 2
     environment:
@@ -172,7 +182,7 @@ services:
     !!merge <<: [*build-js, *deploy]
     environment:
       !!merge <<: *environment
-      SUPERBLOCKS_AGENT_PROCESS_ASYNC: "true"
+      SUPERBLOCKS_AGENT_PROCESS_ASYNC: 'true'
       SUPERBLOCKS_AGENT_PLUGIN_EVENTS: execute
       SUPERBLOCKS_AGENT_BUCKET: BE
       SUPERBLOCKS_AGENT_REDIS_GROUP: ten
@@ -182,7 +192,7 @@ services:
     !!merge <<: [*build-js, *deploy]
     environment:
       !!merge <<: *environment
-      SUPERBLOCKS_AGENT_PROCESS_ASYNC: "true"
+      SUPERBLOCKS_AGENT_PROCESS_ASYNC: 'true'
       SUPERBLOCKS_AGENT_PLUGIN_EVENTS: test
       SUPERBLOCKS_AGENT_BUCKET: BA
       SUPERBLOCKS_AGENT_REDIS_GROUP: eleven
@@ -207,21 +217,21 @@ services:
   golang.ba.all:
     !!merge <<: *build-go
     environment:
-      !!merge <<: [*environment-golang]
+      !!merge <<: *environment-golang
       SUPERBLOCKS_WORKER_GO_WORKER_BUCKET: BA
     depends_on: [redis]
   golang.be.all:
     !!merge <<: *build-go
     environment:
-      !!merge <<: [*environment-golang]
+      !!merge <<: *environment-golang
       SUPERBLOCKS_WORKER_GO_WORKER_BUCKET: BE
     depends_on: [redis]
   mock_oauth_server:
     image: ghcr.io/navikt/mock-oauth2-server:2.1.0
     environment:
       SERVER_PORT: 5678
     ports:
-      - "5678:5678"
+      - 5678:5678
   echo:
     image: ghcr.io/superblocksteam/docker.io/ealen/echo-server:0.9.2
 volumes:

internal/registration/registration.go

@@ -34,6 +34,7 @@ var SUPERBLOCKS_PLUGIN_VERSIONS = map[string][]string{
 	"gsheets":            {"0.0.18"},
 	"kafka":              {"0.0.1"},
 	"javascript":         {"0.0.9"},
+	"kinesis":            {"0.0.1"},
 	"mariadb":            {"0.0.11"},
 	"mongodb":            {"0.0.7"},
 	"mssql":              {"0.0.10"},

internal/registration/registration_test.go

@@ -65,5 +65,5 @@ func TestRegistrationRequest(t *testing.T) {
 	assert.Equal(t, "https://foo.bar.com/api/v1/agents/register", request.URL.String())
 	assert.Equal(t, http.MethodPost, request.Method)
 	assert.Equal(t, expectedRequestHeaders, request.Header)
-	assert.Equal(t, "{\"pluginVersions\":{\"athena\":[\"0.0.1\"],\"bigquery\":[\"0.0.7\"],\"cockroachdb\":[\"0.0.2\"],\"confluent\":[\"0.0.1\"],\"cosmosdb\":[\"0.0.1\"],\"couchbase\":[\"0.0.2\"],\"databricks\":[\"0.0.1\"],\"dynamodb\":[\"0.0.7\"],\"email\":[\"0.0.7\"],\"gcs\":[\"0.0.1\"],\"graphql\":[\"0.0.8\"],\"gsheets\":[\"0.0.18\"],\"javascript\":[\"0.0.9\"],\"kafka\":[\"0.0.1\"],\"mariadb\":[\"0.0.11\"],\"mongodb\":[\"0.0.7\"],\"mssql\":[\"0.0.10\"],\"mysql\":[\"0.0.11\"],\"openai\":[\"0.0.3\"],\"oracledb\":[\"0.0.1\"],\"postgres\":[\"0.0.11\"],\"python\":[\"0.0.7\"],\"redis\":[\"0.0.1\"],\"redpanda\":[\"0.0.1\"],\"redshift\":[\"0.0.7\"],\"restapi\":[\"0.0.12\"],\"restapiintegration\":[\"0.0.13\"],\"rockset\":[\"0.0.7\"],\"s3\":[\"0.0.10\"],\"salesforce\":[\"0.0.1\"],\"smtp\":[\"0.0.1\"],\"snowflake\":[\"0.0.7\"],\"superblocks-ocr\":[\"0.0.1\"],\"workflow\":[\"0.0.4\"]},\"type\":2,\"tags\":{\"foo\":[\"bar\"]},\"signingKeyId\":\"123\",\"verificationKeyIds\":[\"123\"],\"verificationKeys\":{\"123\":\"key1\"}}\n", string(body))
+	assert.Equal(t, "{\"pluginVersions\":{\"athena\":[\"0.0.1\"],\"bigquery\":[\"0.0.7\"],\"cockroachdb\":[\"0.0.2\"],\"confluent\":[\"0.0.1\"],\"cosmosdb\":[\"0.0.1\"],\"couchbase\":[\"0.0.2\"],\"databricks\":[\"0.0.1\"],\"dynamodb\":[\"0.0.7\"],\"email\":[\"0.0.7\"],\"gcs\":[\"0.0.1\"],\"graphql\":[\"0.0.8\"],\"gsheets\":[\"0.0.18\"],\"javascript\":[\"0.0.9\"],\"kafka\":[\"0.0.1\"],\"kinesis\":[\"0.0.1\"],\"mariadb\":[\"0.0.11\"],\"mongodb\":[\"0.0.7\"],\"mssql\":[\"0.0.10\"],\"mysql\":[\"0.0.11\"],\"openai\":[\"0.0.3\"],\"oracledb\":[\"0.0.1\"],\"postgres\":[\"0.0.11\"],\"python\":[\"0.0.7\"],\"redis\":[\"0.0.1\"],\"redpanda\":[\"0.0.1\"],\"redshift\":[\"0.0.7\"],\"restapi\":[\"0.0.12\"],\"restapiintegration\":[\"0.0.13\"],\"rockset\":[\"0.0.7\"],\"s3\":[\"0.0.10\"],\"salesforce\":[\"0.0.1\"],\"smtp\":[\"0.0.1\"],\"snowflake\":[\"0.0.7\"],\"superblocks-ocr\":[\"0.0.1\"],\"workflow\":[\"0.0.4\"]},\"type\":2,\"tags\":{\"foo\":[\"bar\"]},\"signingKeyId\":\"123\",\"verificationKeyIds\":[\"123\"],\"verificationKeys\":{\"123\":\"key1\"}}\n", string(body))
 }

pkg/plugin/plugin_test.go

@@ -64,6 +64,8 @@ func TestMarshal(t *testing.T) {
 			built, err = step.GetConfig().(*apiv1.Step_Restapiintegration).Build()
 		} else if pluginId == `"redshift"` {
 			built, err = step.GetConfig().(*apiv1.Step_Redshift).Build()
+		} else if pluginId == `"kinesis"` {
+			built, err = step.GetConfig().(*apiv1.Step_Kinesis).Build()
 		} else if pluginId == `"mysql"` {
 			built, err = step.GetConfig().(*apiv1.Step_Mysql).Build()
 		} else if pluginId == `"mssql"` {