From 665c7c1b7b22cd6cf7e308f50a361a23d1e94cc9 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Wed, 6 Nov 2024 14:24:17 +0000 Subject: [PATCH 01/10] DOC-686 Add MVP docs for Redpanda Connect Helm chart --- .github/workflows/generate-helm-spec.yml | 93 +++ modules/ROOT/nav.adoc | 8 +- .../ROOT/pages/get-started/helm-chart.adoc | 361 +++++++++++ modules/ROOT/pages/get-started/index.adoc | 5 + .../pages/get-started/rpk.adoc} | 10 +- modules/reference/pages/index.adoc | 3 + .../reference/pages/k-connect-helm-spec.adoc | 585 ++++++++++++++++++ 7 files changed, 1058 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/generate-helm-spec.yml create mode 100644 modules/ROOT/pages/get-started/helm-chart.adoc create mode 100644 modules/ROOT/pages/get-started/index.adoc rename modules/{guides/pages/getting_started.adoc => ROOT/pages/get-started/rpk.adoc} (93%) create mode 100644 modules/reference/pages/index.adoc create mode 100644 modules/reference/pages/k-connect-helm-spec.adoc diff --git a/.github/workflows/generate-helm-spec.yml b/.github/workflows/generate-helm-spec.yml new file mode 100644 index 00000000..ee0fa80f --- /dev/null +++ b/.github/workflows/generate-helm-spec.yml @@ -0,0 +1,93 @@ +--- + name: Generate Helm Spec Docs + on: + repository_dispatch: # Allows other repositories to trigger this workflow + types: [generate-helm-spec-docs] + workflow_dispatch: # Allows manual trigger of the workflow + jobs: + trigger: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.RP_AWS_CRED_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }} + - uses: aws-actions/aws-secretsmanager-get-secrets@v2 + with: + secret-ids: | + ,sdlc/prod/github/actions_bot_token + parse-json-secrets: true + # Download the helm-docs tool which generates documentation from Helm charts. + - name: Download helm-docs + run: | + curl -sL https://github.com/norwoodj/helm-docs/releases/download/v1.14.2/helm-docs_1.14.2_Linux_x86_64.tar.gz | tar xz + # Add the helm-docs tool to the system's PATH for easy access. + - name: Add helm-docs to $PATH + run: | + echo "helm-docs" >> $GITHUB_PATH + sudo mv helm-docs /usr/local/bin/ + # Checkout the main branch of the helm-charts repository. + - name: Checkout helm-charts repository + uses: actions/checkout@v4 + with: + repository: redpanda-data/helm-charts + ref: main + path: helm-charts + token: ${{ env.ACTIONS_BOT_TOKEN }} + # Generate the Helm documentation using the helm-docs tool. + - name: Generate Helm docs + run: | + helm-docs + working-directory: ./helm-charts + # Install pandoc, a tool to convert between different markup formats. + - name: Install pandoc + run: | + sudo apt-get install pandoc + # Checkout the main branch of the redpanda-docs repository. + - name: Checkout documentation repository + uses: actions/checkout@v4 + with: + repository: redpanda-data/rp-connect-docs + ref: main + path: redpanda-docs + token: ${{ env.ACTIONS_BOT_TOKEN }} + # Convert the generated Markdown Helm documentation to AsciiDoc format using pandoc. + - name: Convert Markdown to AsciiDoc + run: | + pandoc ./helm-charts/charts/connect/README.md -t asciidoc -o ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + - name: Modify third-level headings format + run: | + sed -i 's/\(\[[0-9]*\)\]\./\1\\]\./g' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + sed -i 's/=== \(http\([^[]\|\%5[BbDd]\)*\)\[\([^]]*\)\]/=== link:++\1++\[\3\]/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + - name: Correct AsciiDoc format + run: | + # Correct the title format + sed -i 's/^== # \(.*\)/= \1/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + # Correct the description format + sed -i 's/^== description: \(.*\)/:description: \1/' ./redpanda-docs/modules/reference/pages/k-connect-helm-spec.adoc + # Check if any changes were made in the documentation. + - name: Check if changes were made + id: check_changes + run: | + cd ./redpanda-docs + changes=$(git status --porcelain) + if [ -z "$changes" ]; then + echo "has_changes=false" >> $GITHUB_ENV + else + echo "has_changes=true" >> $GITHUB_ENV + fi + # If changes were detected, commit those changes. + - name: Commit changes + if: env.has_changes == 'true' + run: | + cd ./redpanda-docs + git config --global user.email "vbotbuildovich@users.noreply.github.com" + git config --global user.name "vbotbuildovich" + git add modules/reference/* + git commit -m "auto-docs: Update Helm spec" + git push origin main + env: + ACCESS_TOKEN: ${{ env.ACTIONS_BOT_TOKEN }} \ No newline at end of file diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index 8fd72e8b..e891de9a 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -1,6 +1,8 @@ * xref:about.adoc[] ** xref:whats_new_rpcn.adoc[] - +** xref:ROOT:get-started/index.adoc[Get Started] +*** xref:ROOT:get-started/rpk.adoc[Redpanda `rpk` CLI] +*** xref:ROOT:get-started/helm-chart.adoc[Kubernetes] * xref:configuration:about.adoc[] ** xref:configuration:resources.adoc[] ** xref:configuration:batching.adoc[] @@ -295,7 +297,6 @@ ** xref:components:logger/about.adoc[] * xref:guides:index.adoc[] -** xref:guides:getting_started.adoc[] ** xref:guides:bloblang/about.adoc[] *** xref:guides:bloblang/walkthrough.adoc[] *** xref:guides:bloblang/functions.adoc[] @@ -326,3 +327,6 @@ ** xref:cookbooks:joining_streams.adoc[] ** xref:cookbooks:rag.adoc[] ** xref:cookbooks:redpanda_migrator.adoc[] +** xref:reference:k-connect-helm-spec.adoc[] +* xref:reference:index.adoc[] +** xref:reference:k-connect-helm-spec.adoc[Helm Chart Specification] diff --git a/modules/ROOT/pages/get-started/helm-chart.adoc b/modules/ROOT/pages/get-started/helm-chart.adoc new file mode 100644 index 00000000..d667e2d3 --- /dev/null +++ b/modules/ROOT/pages/get-started/helm-chart.adoc @@ -0,0 +1,361 @@ += Get Started with the Redpanda Connect Helm Chart +:description: Deploy Redpanda Connect on Kubernetes using Helm. +:page-aliases: getting_started:kubernetes_install.adoc, ROOT:kubernetes_install.adoc + +This guide explains how to deploy and configure Redpanda Connect on Kubernetes using the Helm chart. + +Redpanda Connect is a powerful stream processor that integrates data across various glossterm:source connector[sources] (inputs) and glossterm:sink connector[sinks] (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it a versatile solution for data pipelines that require high performance and resilience. + +Common use cases for Redpanda Connect include: + +* **Real-time data ingestion**: Ingesting data from various sources into Redpanda or other data platforms. +* **Data transformation**: Enriching and transforming data to match business requirements before forwarding it to a destination. +* **Data filtering and routing**: Filtering data and routing it to the appropriate sinks based on predefined conditions. + +== Prerequisites + +* A running Kubernetes cluster. +* `kubectl` CLI and `helm` CLI installed. + +== Install with Helm + +To deploy Redpanda Connect on Kubernetes, use the official Helm chart. + +[source,bash] +---- +helm repo add redpanda https://charts.redpanda.com <1> +helm repo update <2> +helm install redpanda-connect redpanda/connect --namespace --create-namespace <3> +---- + +<1> Adds the Redpanda Helm repository. +<2> Updates your local Helm repository cache. +<3> Installs Redpanda Connect in the given namespace. You can customize this deployment by configuring values in the Helm chart. + +== Run example pipelines + +Redpanda Connect can run in two different modes: **standalone mode** and **streams mode**. + +* **Standalone mode**: Allows you to run a single pipeline configuration at a time, making it suitable for simpler, fixed configurations that don't need to run concurrently. +* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing needs and scenarios requiring multiple concurrent data flows. + +=== Use standalone mode + +In standalone mode, all configurations are specified directly within `config` in `values.yaml`. This approach is best for running a single pipeline at a time. + +. Create a ConfigMap to provide data that Redpanda Connect can read. This example ConfigMap contains a JSON object simulating input data: ++ +[source,bash] +---- +kubectl create configmap connect-input --from-literal=input-data='{"name": "Test User", "email": "test.user@example.com"}' --namespace +---- + +. Create a `pipeline.yaml` file and add the following pipeline configuration for Redpanda Connect. This file configures Redpanda Connect in standalone mode to read from the file mounted by the ConfigMap and convert the usernames to uppercase. ++ +.`pipeline.yaml` +[source,yaml] +---- +extraVolumes: + - name: input-config + configMap: + name: connect-input +extraVolumeMounts: + - name: input-config + mountPath: /input + subPath: input-data +config: + input: + file: + paths: + - "/input" <1> + pipeline: + processors: + - mapping: | + root.name = this.name.uppercase() <2> + output: + stdout: {} <3> +---- ++ +<1> For the input, use the contents of the file at the path where the ConfigMap data is mounted. +<2> Convert the name field in the input JSON to uppercase using xref:guides:bloblang/walkthrough.adoc[Bloblang], the native mapping language of Redpanda Connect. +<3> Output the processed data to stdout, making it viewable in logs. + +. Deploy the pipeline: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Check the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace +---- ++ +You should see the username converted to uppercase in the output: ++ +[,json] +---- +{"name":"TEST USER"} +---- + +. Check the Pod's status: ++ +[source,bash] +---- +kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +---- ++ +After a few moments, the Pod enters a **CrashLoopBackOff** status because containers are expected to run continuously. When Redpanda Connect finishes processing the input file and has no further data to process, the Pod exits, causing Kubernetes to restart it repeatedly. ++ +To prevent this status, configure Redpanda Connect to continue processing data indefinitely. One way to do this is by using a `generate` input instead of a static input file. + +. Update the `pipeline.yaml` file to use the `generate` input to produce data every second: ++ +[source,yaml] +---- +config: + input: + generate: + interval: 1s + count: 0 # Setting count to 0 ensures it generates data indefinitely. + mapping: | + let first_name = fake("first_name") + let last_name = fake("last_name") + root.user_id = counter() + root.name = ($first_name + " " + $last_name) + root.email = ($first_name.slice(0,1) + $last_name + "@example.com") + root.timestamp = now() + pipeline: + processors: + - mapping: | + root.name = this.name.uppercase() + output: + stdout: {} +---- ++ +This configuration generates a JSON object with: ++ +- `user_id`: A unique identifier for each record, generated using the `counter()` function. +- `name`: A randomly generated first and last name, using the `fake()` function. +- `email`: An email based on the generated names. +- `timestamp`: The current timestamp at the time of generation, using the `now()` function. + +. Deploy the updated configuration: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see logs showing JSON objects similar to the following, with names in uppercase: ++ +[source,json,role="no-copy"] +---- +{"name":"ZOIE SIPES"} +{"name":"LORENA KERTZMANN"} +{"name":"DALLAS BOYER"} +{"name":"LOUIE WILDERMAN"} +{"name":"EMILIA KOEPP"} +{"name":"KALEIGH PACOCHA"} +---- + +. Check the Pod's status: ++ +[source,bash] +---- +kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +---- ++ +The pod should now be running without entering a CrashLoopBackOff state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. + +=== Use streams mode + +Streams mode is designed for running multiple pipelines at once. In streams mode, configurations are managed through a Kubernetes ConfigMap. + +. Define your pipeline configurations in the following separate YAML files: ++ +.`woof.yaml` +[source,yaml] +---- +input: + generate: + mapping: root = "woof" # Generates a message with the word "woof" at regular intervals. + interval: 5s + count: 0 +output: + stdout: + codec: lines # Outputs each message as a new line in stdout. +---- ++ +.`meow.yaml` +[source,yaml] +---- +input: + generate: + mapping: root = "meow" # Generates a message with the word "meow" at regular intervals. + interval: 2s + count: 0 +output: + stdout: + codec: lines # Outputs each message as a new line in stdout. +---- + +. Bundle the configuration files into a ConfigMap, which Redpanda Connect will reference: ++ +[source,bash] +---- +kubectl create configmap connect-streams --from-file=woof.yaml --from-file=meow.yaml --namespace +---- + +. Configure Redpanda Connect in streams mode and specify the name of the ConfigMap to use: ++ +.`connect.yaml` +[source,yaml] +---- +streams: + enabled: true <1> + streamsConfigMap: "connect-streams" <2> +---- ++ +<1> Enable streams mode in Redpanda Connect. +<2> Use the given ConfigMap as the pipeline configuration. + +. Deploy the chart: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values connect.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see logs showing a combination of outputs from both pipelines: ++ +[.no-copy] +---- +woof +meow +meow +meow +woof +meow +meow +---- + +=== Update the pipeline in streams mode + +To update a pipeline in streams mode: + +. Modify one of the configuration files locally. ++ +.`woof.yaml` +[source,yaml] +---- +# Updated woof.yaml +input: + generate: + mapping: root = "bark" # Updated to generate a message with the word "bark" instead of "woof." + interval: 5s + count: 0 +output: + stdout: + codec: lines +---- + +. Update the ConfigMap with the modified file: ++ +[source,bash] +---- +kubectl create configmap connect-streams --from-file=woof.yaml --from-file=meow.yaml --namespace --dry-run=client -o yaml | kubectl apply -f - +---- + +. Restart the Deployment: ++ +[source,bash] +---- +kubectl rollout restart deployment/redpanda-connect --namespace +---- + +=== Global configuration + +When deploying Redpanda Connect in streams mode, you can configure global tracing, logging, and HTTP settings to apply across all pipelines. Specify these in your `values.yaml` overrides under the `metrics`, `logger`, and `tracing` sections. + +[source,yaml] +---- +metrics: + prometheus: {} # Enable Prometheus metrics collection. + +tracing: + openTelemetry: + http: [] # Configure OpenTelemetry HTTP tracing. + grpc: [] + tags: {} + +logger: + level: INFO # Set logging level (e.g., INFO, DEBUG). + static_fields: + '@service': redpanda-connect # Add static fields to logs for better traceability. +---- + +== Access the HTTP server on Redpanda Connect + +To manage and monitor Redpanda Connect, you can use its HTTP server, which provides useful endpoints for version checking, pipeline management, and more. By default, Redpanda Connect exposes this server using a Kubernetes ClusterIP Service, accessible only within the cluster. + +. Forward the ports of the ClusterIP Service to your local device: ++ +[source,bash] +---- +kubectl port-forward svc/redpanda-connect 8080:80 --namespace +---- + +. Access the HTTP server locally. For example, to check the Redpanda Connect version, run: ++ +[,bash] +---- +curl http://localhost:8080/version +---- ++ +Example output: ++ +[,json] +---- +{ + "version": "v4.38.0", + "built": "2024-10-17T09:27:42Z" +} +---- + +You can also configure external access using a LoadBalancer Service or an Ingress. See the link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml#L79C1-L107C31[Helm values] for more details. + +== Next steps + +* Learn more about xref:guides:bloblang/walkthrough.adoc[Bloblang], the mapping language for processing data in Redpanda Connect. +* Try more hands-on examples with one of the xref:cookbooks:index.adoc[Cookbooks]. + +== Suggested reading + +* link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[Helm values] +* xref:components:http/about.adoc[HTTP server] +* xref:components:inputs/about.adoc[Inputs] +* xref:components:processors/about.adoc[Processors] +* xref:components:outputs/about.adoc[Outputs] +* xref:guides:monitoring.adoc[Monitoring] +* xref:configuration:about.adoc[More about configuration] diff --git a/modules/ROOT/pages/get-started/index.adoc b/modules/ROOT/pages/get-started/index.adoc new file mode 100644 index 00000000..41f09b7b --- /dev/null +++ b/modules/ROOT/pages/get-started/index.adoc @@ -0,0 +1,5 @@ += Get Started with Redpanda Connect +:description: Choose from one of the following deployment options to learn how to install Redpanda Connect and deploy your first pipeline. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/guides/pages/getting_started.adoc b/modules/ROOT/pages/get-started/rpk.adoc similarity index 93% rename from modules/guides/pages/getting_started.adoc rename to modules/ROOT/pages/get-started/rpk.adoc index b600d98a..0ff9687a 100644 --- a/modules/guides/pages/getting_started.adoc +++ b/modules/ROOT/pages/get-started/rpk.adoc @@ -1,8 +1,8 @@ -= Get Started -:description: Getting started with Redpanda Connect. -:page-aliases: getting_started:overview.adoc, ROOT:install.adoc += Get Started with Redpanda Connect using `rpk` +:description: pass:q[Deploy your first pipelines using Redpanda Connect and `rpk`.] +:page-aliases: getting_started:overview.adoc, ROOT:install.adoc, guides:getting_started.adoc -This topic explains how to get started with Redpanda Connect. +This topic explains how to get started with Redpanda Connect using `rpk`, the Redpanda CLI. == Install @@ -31,7 +31,7 @@ To use `rpk` in Docker, you can use the Redpanda Docker image, which includes bo docker pull docker.redpanda.com/redpandadata/connect ---- -== Run +== Run a pipeline A Redpanda Connect stream pipeline is configured with a single xref:configuration:about.adoc[config file], you can generate a fresh one with: diff --git a/modules/reference/pages/index.adoc b/modules/reference/pages/index.adoc new file mode 100644 index 00000000..8e1a7f85 --- /dev/null +++ b/modules/reference/pages/index.adoc @@ -0,0 +1,3 @@ += Reference +:description: The following topics provide reference material for Redpanda Connect. +:page-layout: index \ No newline at end of file diff --git a/modules/reference/pages/k-connect-helm-spec.adoc b/modules/reference/pages/k-connect-helm-spec.adoc new file mode 100644 index 00000000..3aef3085 --- /dev/null +++ b/modules/reference/pages/k-connect-helm-spec.adoc @@ -0,0 +1,585 @@ += Redpanda Connect Chart Specification +:description: Find the default values and descriptions of settings in the Redpanda Connect Helm chart. + +image:https://img.shields.io/badge/Version-3.0.0-informational?style=flat-square[Version: +3.0.0] +image:https://img.shields.io/badge/Type-application-informational?style=flat-square[Type: +application] +image:https://img.shields.io/badge/AppVersion-4.38.0-informational?style=flat-square[AppVersion: +4.38.0] + +Redpanda Connect is a high performance and resilient stream processor, +able to connect various sources and sinks in a range of brokering +patterns and perform hydration, enrichments, transformations and filters +on payloads. + +This Helm chart deploys a Redpanda Connect instance in either streams +mode or standalone. + +This page describes the contents of the chart's +https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[`values.yaml` +file]. Each of the settings is listed and described on this page, along +with any default values. + +For instructions on how to install and use the chart, including how to +override and customize the chart’s values, refer to the +https://docs.redpanda.com/redpanda-connect/get-started/helm-chart/[deployment +documentation]. + +NOTE: To migrate to this chart from the +https://github.com/redpanda-data/redpanda-connect-helm-chart[Benthos +Helm chart], see the +https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/MIGRATION_FROM_BENTHOS.md[migration +guide in this GitHub repository]. + +''''' + +Autogenerated from chart metadata using +https://github.com/norwoodj/helm-docs/releases/v1.14.2[helm-docs +v1.14.2] + +== Source Code + +* https://github.com/redpanda-data/helm-charts + +== Settings + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=affinity[affinity] + +Affinity rules for scheduling Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=args[args] + +Override the default arguments for the entrypoint command. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling[autoscaling] + +Configuration for Horizontal Pod Autoscaling for the Redpanda Connect +Deployment. + +*Default:* + +.... +{"enabled":false,"maxReplicas":12,"metrics":[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}],"minReplicas":1} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.enabled[autoscaling.enabled] + +Enable Horizontal Pod Autoscaling. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.maxReplicas[autoscaling.maxReplicas] + +Maximum number of replicas if autoscaling is enabled. + +*Default:* `12` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.metrics[autoscaling.metrics] + +List of metrics to use for scaling, such as CPU utilization. + +*Default:* + +.... +[{"resource":{"name":"cpu","target":{"averageUtilization":80,"type":"Utilization"}},"type":"Resource"}] +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=autoscaling.minReplicas[autoscaling.minReplicas] + +Minimum number of replicas if autoscaling is enabled. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=command[command] + +Override the default entrypoint command of the container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=commonLabels[commonLabels] + +Additional labels to apply to all resources created by this chart. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=config[config] + +Configuration for `/redpanda-connect.yaml` file. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.annotations[deployment.annotations] + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe[deployment.livenessProbe] + +Configuration for the liveness probe that checks if the container is +running. + +*Default:* + +.... +{"failureThreshold":3,"httpGet":{"path":"/ping","port":"http"},"periodSeconds":5,"successThreshold":1,"timeoutSeconds":2} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.failureThreshold[deployment.livenessProbe.failureThreshold] + +Number of consecutive failures before Pod is restarted. + +*Default:* `3` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.httpGet[deployment.livenessProbe.httpGet] + +HTTP GET request configuration for the liveness probe. + +*Default:* + +.... +{"path":"/ping","port":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.periodSeconds[deployment.livenessProbe.periodSeconds] + +Frequency (in seconds) of liveness checks. + +*Default:* `5` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.successThreshold[deployment.livenessProbe.successThreshold] + +Number of consecutive successes required to mark as healthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.livenessProbe.timeoutSeconds[deployment.livenessProbe.timeoutSeconds] + +Timeout (in seconds) for each liveness check. + +*Default:* `2` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.podAnnotations[deployment.podAnnotations] + +Annotations to add to Redpanda Connect Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.podLabels[deployment.podLabels] + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe[deployment.readinessProbe] + +Configuration for the readiness probe that checks if the container is +ready to accept traffic. + +*Default:* + +.... +{"failureThreshold":1,"httpGet":{"path":"/ready","port":"http"},"periodSeconds":5,"successThreshold":1,"timeoutSeconds":2} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.failureThreshold[deployment.readinessProbe.failureThreshold] + +Number of consecutive failures before marking Pod as unhealthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.httpGet[deployment.readinessProbe.httpGet] + +HTTP GET request configuration for the readiness probe. + +*Default:* + +.... +{"path":"/ready","port":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.periodSeconds[deployment.readinessProbe.periodSeconds] + +Frequency (in seconds) of readiness checks. + +*Default:* `5` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.successThreshold[deployment.readinessProbe.successThreshold] + +Number of consecutive successes before marking Pod as healthy. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.readinessProbe.timeoutSeconds[deployment.readinessProbe.timeoutSeconds] + +Timeout (in seconds) for each readiness check. + +*Default:* `2` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.replicaCount[deployment.replicaCount] + +Number of Replicas for the Redpanda Connect Deployment. + +*Default:* `1` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.restartPolicy[deployment.restartPolicy] + +Restart policy for containers in the Pod. + +*Default:* `"Always"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.rolloutConfigMap[deployment.rolloutConfigMap] + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.terminationGracePeriodSeconds[deployment.terminationGracePeriodSeconds] + +*Default:* `60` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=env[env] + +Environment variables to pass to the Redpanda Connect container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=envFrom[envFrom] + +Names of Secrets or ConfigMaps that contain environment variables to +pass to the Redpanda Connect container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=extraVolumeMounts[extraVolumeMounts] + +Additional volume mounts for the container. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=extraVolumes[extraVolumes] + +Additional volumes to add to the Pod. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=fullnameOverride[fullnameOverride] + +Override for the full name template. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http[http] + +Configuration for the HTTP server used to expose Redpanda Connect +endpoints. + +*Default:* + +.... +{"address":"0.0.0.0:4195","cors":{"enabled":false},"debug_endpoints":false,"enabled":true,"root_path":"/redpanda-connect"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.address[http.address] + +Bind Address for the HTTP server. + +*Default:* `"0.0.0.0:4195"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.cors.enabled[http.cors.enabled] + +Enable CORS support for HTTP server. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.debug_endpoints[http.debug_endpoints] + +Enable debug endpoints for the HTTP server. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.enabled[http.enabled] + +Enable HTTP server. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=http.root_path[http.root_path] + +Root path for HTTP API endpoints. + +*Default:* `"/redpanda-connect"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image[image] + +Configuration for the Redpanda Connect container image. + +*Default:* + +.... +{"pullPolicy":"IfNotPresent","repository":"docker.redpanda.com/redpandadata/connect","tag":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.pullPolicy[image.pullPolicy] + +Image pull policy, such as `IfNotPresent` or `Always`. + +*Default:* `"IfNotPresent"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.repository[image.repository] + +Docker repository for the Redpanda Connect image. + +*Default:* + +.... +"docker.redpanda.com/redpandadata/connect" +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=image.tag[image.tag] + +Override for the default image tag (uses chart `appVersion` if empty). + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=imagePullSecrets[imagePullSecrets] + +List of Secrets for private Docker registry authentication. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.annotations[ingress.annotations] + +Annotations for the Ingress resource. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.className[ingress.className] + +Ingress Class name for the Ingress resource. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.enabled[ingress.enabled] + +Enable Ingress resource creation. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.hosts[ingress.hosts] + +List of host definitions for the Ingress resource. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=ingress.tls[ingress.tls] + +TLS configuration for Ingress resource. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=initContainers[initContainers] + +Additional init containers to add to the Deployment. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=nameOverride[nameOverride] + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=nodeSelector[nodeSelector] + +Node selector for scheduling Pods. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podDisruptionBudget[podDisruptionBudget] + +Configuration for the PodDisruptionBudget, which controls voluntary +evictions of Redpanda Connect Pods. + +*Default:* `{"enabled":false}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podDisruptionBudget.enabled[podDisruptionBudget.enabled] + +Enable a PodDisruptionBudget for the Deployment. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=podSecurityContext[podSecurityContext] + +Security context to be applied at the Pod level. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=resources[resources] + +Resource limits and requests for the container. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=securityContext[securityContext] + +Security context to be applied to containers in the Pod. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service[service] + +Configuration for the Kubernetes Service that exposes Redpanda Connect. + +*Default:* + +.... +{"extraPorts":null,"name":"http","port":80,"protocol":"TCP","targetPort":"http","type":"ClusterIP"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.extraPorts[service.extraPorts] + +Additional ports to expose through the Service. + +*Default:* `nil` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.name[service.name] + +Name of the Kubernetes Service. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.port[service.port] + +Service port for accessing the application. + +*Default:* `80` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.protocol[service.protocol] + +Protocol to use for the Service, for example TCP or UDP. + +*Default:* `"TCP"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.targetPort[service.targetPort] + +Target port on the container. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=service.type[service.type] + +Type of Kubernetes Service, for example ClusterIP, NodePort, or +LoadBalancer. + +*Default:* `"ClusterIP"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount[serviceAccount] + +Configuration for the Kubernetes ServiceAccount associated with the +Redpanda Connect Pods. + +*Default:* + +.... +{"annotations":{},"create":true,"name":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.annotations[serviceAccount.annotations] + +Annotations to add to the ServiceAccount. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.create[serviceAccount.create] + +Specify whether a ServiceAccount should be created. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceAccount.name[serviceAccount.name] + +Name of the ServiceAccount to use. If not set and `create` is `true`, a +name is autogenerated. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor[serviceMonitor] + +Configuration for the ServiceMonitor used by Prometheus to scrape +Redpanda Connect metrics. + +*Default:* + +.... +{"enabled":false,"interval":"10s","scheme":"http"} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.enabled[serviceMonitor.enabled] + +Enable the creation of a ServiceMonitor for Prometheus monitoring. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.interval[serviceMonitor.interval] + +Scrape interval for Prometheus monitoring. + +*Default:* `"10s"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=serviceMonitor.scheme[serviceMonitor.scheme] + +HTTP Scheme to use for scraping. + +*Default:* `"http"` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams[streams] + +Configuration settings for the streams mode feature in Redpanda Connect. + +*Default:* + +.... +{"api":{"enable":true},"enabled":false,"streamsConfigMap":""} +.... + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.api.enable[streams.api.enable] + +Enable the streams API. + +*Default:* `true` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.enabled[streams.enabled] + +Enable streams mode. + +*Default:* `false` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=streams.streamsConfigMap[streams.streamsConfigMap] + +Name of the ConfigMap that contains stream configuration files. + +*Default:* `""` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=tolerations[tolerations] + +Tolerations for scheduling Pods onto nodes with matching taints. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=topologySpreadConstraints[topologySpreadConstraints] + +Spread incoming Pods across the cluster based on topology. + +*Default:* `[]` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=updateStrategy[updateStrategy] + +Update strategy for the Deployment, for example `RollingUpdate`. + +*Default:* `{}` + +=== https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=watch[watch] + +Experimental feature: Watch config files for changes and automatically +apply them without needing to rollout the Deployment. Make sure that +`deployment.rolloutConfigMap` is `false`. + +*Default:* `false` From 2505b648be2faa0aaa1c62d04ef60b9d630e70a8 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 7 Nov 2024 10:17:22 +0000 Subject: [PATCH 02/10] Add Get Started --- antora.yml | 2 +- local-antora-playbook.yml | 2 +- modules/ROOT/nav.adoc | 16 ++++++++++------ modules/ROOT/pages/get-started/index.adoc | 5 ----- modules/{ROOT => get-started}/pages/about.adoc | 2 +- modules/get-started/pages/index.adoc | 5 +++++ .../pages/quickstarts}/helm-chart.adoc | 4 ++-- modules/get-started/pages/quickstarts/index.adoc | 5 +++++ .../pages/quickstarts}/rpk.adoc | 4 ++-- .../pages/whats-new.adoc} | 1 + modules/reference/pages/k-connect-helm-spec.adoc | 6 +++++- 11 files changed, 33 insertions(+), 19 deletions(-) delete mode 100644 modules/ROOT/pages/get-started/index.adoc rename modules/{ROOT => get-started}/pages/about.adoc (93%) create mode 100644 modules/get-started/pages/index.adoc rename modules/{ROOT/pages/get-started => get-started/pages/quickstarts}/helm-chart.adoc (96%) create mode 100644 modules/get-started/pages/quickstarts/index.adoc rename modules/{ROOT/pages/get-started => get-started/pages/quickstarts}/rpk.adoc (93%) rename modules/{ROOT/pages/whats_new_rpcn.adoc => get-started/pages/whats-new.adoc} (99%) diff --git a/antora.yml b/antora.yml index 8a74c7f9..746d301b 100644 --- a/antora.yml +++ b/antora.yml @@ -1,7 +1,7 @@ name: redpanda-connect title: Connect version: ~ -start_page: ROOT:about.adoc +start_page: get-started:about.adoc nav: - modules/ROOT/nav.adoc asciidoc: diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index f3ddc802..9e89777e 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -1,6 +1,6 @@ site: title: Redpanda Docs - start_page: redpanda-connect:ROOT:about.adoc + start_page: redpanda-connect:get-started:about.adoc url: http://localhost:5002 robots: disallow urls: diff --git a/modules/ROOT/nav.adoc b/modules/ROOT/nav.adoc index e891de9a..a5222f90 100644 --- a/modules/ROOT/nav.adoc +++ b/modules/ROOT/nav.adoc @@ -1,8 +1,11 @@ -* xref:about.adoc[] -** xref:whats_new_rpcn.adoc[] -** xref:ROOT:get-started/index.adoc[Get Started] -*** xref:ROOT:get-started/rpk.adoc[Redpanda `rpk` CLI] -*** xref:ROOT:get-started/helm-chart.adoc[Kubernetes] + +* xref:get-started:index.adoc[Get Started] +** xref:get-started:about.adoc[] +** xref:get-started:whats-new.adoc[What's New] +** xref:get-started:quickstarts/index.adoc[Quickstarts] +*** xref:get-started:quickstarts/rpk.adoc[CLI] +*** xref:get-started:quickstarts/helm-chart.adoc[Helm Chart] + * xref:configuration:about.adoc[] ** xref:configuration:resources.adoc[] ** xref:configuration:batching.adoc[] @@ -319,6 +322,7 @@ *** xref:guides:migration/v4.adoc[] *** xref:guides:migration/v3.adoc[] *** xref:guides:migration/v2.adoc[] + * xref:cookbooks:index.adoc[] ** xref:cookbooks:custom_metrics.adoc[] ** xref:cookbooks:discord_bot.adoc[] @@ -327,6 +331,6 @@ ** xref:cookbooks:joining_streams.adoc[] ** xref:cookbooks:rag.adoc[] ** xref:cookbooks:redpanda_migrator.adoc[] -** xref:reference:k-connect-helm-spec.adoc[] + * xref:reference:index.adoc[] ** xref:reference:k-connect-helm-spec.adoc[Helm Chart Specification] diff --git a/modules/ROOT/pages/get-started/index.adoc b/modules/ROOT/pages/get-started/index.adoc deleted file mode 100644 index 41f09b7b..00000000 --- a/modules/ROOT/pages/get-started/index.adoc +++ /dev/null @@ -1,5 +0,0 @@ -= Get Started with Redpanda Connect -:description: Choose from one of the following deployment options to learn how to install Redpanda Connect and deploy your first pipeline. -:page-layout: index - -{description} \ No newline at end of file diff --git a/modules/ROOT/pages/about.adoc b/modules/get-started/pages/about.adoc similarity index 93% rename from modules/ROOT/pages/about.adoc rename to modules/get-started/pages/about.adoc index 6e2e251e..e09ad07b 100644 --- a/modules/ROOT/pages/about.adoc +++ b/modules/get-started/pages/about.adoc @@ -1,5 +1,5 @@ = Introduction to Redpanda Connect -:page-aliases: guides:delivery_guarantee.adoc +:page-aliases: guides:delivery_guarantee.adoc, ROOT:about.adoc Redpanda Connect is a declarative data streaming service that solves a wide range of data engineering problems with simple, chained, stateless xref:components:processors/about.adoc[processing steps]. It implements transaction-based resiliency with back pressure, so when connecting to at-least-once sources and sinks it's able to guarantee at-least-once delivery without needing to persist messages during transit. diff --git a/modules/get-started/pages/index.adoc b/modules/get-started/pages/index.adoc new file mode 100644 index 00000000..11859c48 --- /dev/null +++ b/modules/get-started/pages/index.adoc @@ -0,0 +1,5 @@ += Get Started +:description: Explore these topics to get started with Redpanda Connect. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/ROOT/pages/get-started/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc similarity index 96% rename from modules/ROOT/pages/get-started/helm-chart.adoc rename to modules/get-started/pages/quickstarts/helm-chart.adoc index d667e2d3..1117701e 100644 --- a/modules/ROOT/pages/get-started/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -1,10 +1,10 @@ = Get Started with the Redpanda Connect Helm Chart :description: Deploy Redpanda Connect on Kubernetes using Helm. -:page-aliases: getting_started:kubernetes_install.adoc, ROOT:kubernetes_install.adoc +:page-aliases: get-started:helm-chart.adoc This guide explains how to deploy and configure Redpanda Connect on Kubernetes using the Helm chart. -Redpanda Connect is a powerful stream processor that integrates data across various glossterm:source connector[sources] (inputs) and glossterm:sink connector[sinks] (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it a versatile solution for data pipelines that require high performance and resilience. +Redpanda Connect is a powerful stream processor that integrates data across various sources (inputs) and sinks (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it a versatile solution for data pipelines that require high performance and resilience. Common use cases for Redpanda Connect include: diff --git a/modules/get-started/pages/quickstarts/index.adoc b/modules/get-started/pages/quickstarts/index.adoc new file mode 100644 index 00000000..8225d2c6 --- /dev/null +++ b/modules/get-started/pages/quickstarts/index.adoc @@ -0,0 +1,5 @@ += Quickstarts +:description: Use your preferred tool to spin up a local instance of Redpanda Connect and get hands-on experience building your first pipelines. +:page-layout: index + +{description} \ No newline at end of file diff --git a/modules/ROOT/pages/get-started/rpk.adoc b/modules/get-started/pages/quickstarts/rpk.adoc similarity index 93% rename from modules/ROOT/pages/get-started/rpk.adoc rename to modules/get-started/pages/quickstarts/rpk.adoc index 0ff9687a..f14ec265 100644 --- a/modules/ROOT/pages/get-started/rpk.adoc +++ b/modules/get-started/pages/quickstarts/rpk.adoc @@ -2,11 +2,11 @@ :description: pass:q[Deploy your first pipelines using Redpanda Connect and `rpk`.] :page-aliases: getting_started:overview.adoc, ROOT:install.adoc, guides:getting_started.adoc -This topic explains how to get started with Redpanda Connect using `rpk`, the Redpanda CLI. +This guide explains how to get started with Redpanda Connect using `rpk`, the Redpanda command-line interface (CLI). == Install -You can install Redpanda Connect using the `rpk` command-line tool (CLI). The `rpk` CLI allows you to create and manage data pipelines with Redpanda Connect as well as xref:ROOT:get-started:rpk-install.adoc[interact with Redpanda clusters]. +The `rpk` CLI allows you to create and manage data pipelines with Redpanda Connect as well as xref:ROOT:get-started:rpk-install.adoc[interact with Redpanda clusters]. [%collapsible] .Also interacting with a Redpanda cluster? diff --git a/modules/ROOT/pages/whats_new_rpcn.adoc b/modules/get-started/pages/whats-new.adoc similarity index 99% rename from modules/ROOT/pages/whats_new_rpcn.adoc rename to modules/get-started/pages/whats-new.adoc index ab12f3da..f96c90f0 100644 --- a/modules/ROOT/pages/whats_new_rpcn.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -1,5 +1,6 @@ = What's New in Redpanda Connect :description: Summary of new features in Redpanda Connect. +:page-aliases: ROOT:whats_new_rpcn.adoc This topic includes new content added from version 4.29.0 onwards. diff --git a/modules/reference/pages/k-connect-helm-spec.adoc b/modules/reference/pages/k-connect-helm-spec.adoc index 3aef3085..71a0b87f 100644 --- a/modules/reference/pages/k-connect-helm-spec.adoc +++ b/modules/reference/pages/k-connect-helm-spec.adoc @@ -231,10 +231,14 @@ Restart policy for containers in the Pod. === https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.rolloutConfigMap[deployment.rolloutConfigMap] +Rollout the deployment on ConfigMap changes. + *Default:* `true` === https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=deployment.terminationGracePeriodSeconds[deployment.terminationGracePeriodSeconds] +Time in seconds for Pod termination grace period. + *Default:* `60` === https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=env[env] @@ -566,7 +570,7 @@ Tolerations for scheduling Pods onto nodes with matching taints. === https://artifacthub.io/packages/helm/redpanda-data/connect?modal=values&path=topologySpreadConstraints[topologySpreadConstraints] -Spread incoming Pods across the cluster based on topology. +Spread incoming Pods across the cluster based on topology to improve distribution and fault tolerance. *Default:* `[]` From 6e274a075f276bd281c0a0d9593ec250cdbebad4 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 7 Nov 2024 11:03:43 +0000 Subject: [PATCH 03/10] Improvements --- .../pages/quickstarts/helm-chart.adoc | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 1117701e..0063bb43 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -36,21 +36,29 @@ helm install redpanda-connect redpanda/connect --namespace --create- Redpanda Connect can run in two different modes: **standalone mode** and **streams mode**. -* **Standalone mode**: Allows you to run a single pipeline configuration at a time, making it suitable for simpler, fixed configurations that don't need to run concurrently. -* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing needs and scenarios requiring multiple concurrent data flows. +* **Standalone mode**: Allows you to run a single pipeline at a time, making it suitable for simpler configurations that don't need to run concurrently. +* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing use cases, multiple concurrent data flows. === Use standalone mode -In standalone mode, all configurations are specified directly within `config` in `values.yaml`. This approach is best for running a single pipeline at a time. +In standalone mode, you configure a single pipeline within the `config` section of your Helm values file. This mode is best suited for running a single pipeline from start to finish in one execution. -. Create a ConfigMap to provide data that Redpanda Connect can read. This example ConfigMap contains a JSON object simulating input data: +In this example, you’ll set up a pipeline that reads data from a file, processes it by converting specific fields to uppercase, and then outputs the result to stdout so you can view it in the logs. + +. Create a ConfigMap to provide input data that Redpanda Connect will read from a file. This example ConfigMap contains a JSON object with simulated data, such as a user's name and email: + [source,bash] ---- kubectl create configmap connect-input --from-literal=input-data='{"name": "Test User", "email": "test.user@example.com"}' --namespace ---- ++ +This ConfigMap will act as the source for the file-based input in Redpanda Connect, allowing the pipeline to read and process this structured JSON data. + +. Create a `pipeline.yaml` file. ++ +This file will be used to override the default Helm chart values for Redpanda Connect. -. Create a `pipeline.yaml` file and add the following pipeline configuration for Redpanda Connect. This file configures Redpanda Connect in standalone mode to read from the file mounted by the ConfigMap and convert the usernames to uppercase. +. Add the following configuration to `pipeline.yaml` to set up a simple standalone pipeline that reads data from the file mounted by the ConfigMap, processes it by converting usernames to uppercase, and outputs the result to stdout: + .`pipeline.yaml` [source,yaml] @@ -109,7 +117,7 @@ You should see the username converted to uppercase in the output: kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch ---- + -After a few moments, the Pod enters a **CrashLoopBackOff** status because containers are expected to run continuously. When Redpanda Connect finishes processing the input file and has no further data to process, the Pod exits, causing Kubernetes to restart it repeatedly. +The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the input file and has no further data to process, the Pod exits, causing Kubernetes to restart it repeatedly. + To prevent this status, configure Redpanda Connect to continue processing data indefinitely. One way to do this is by using a `generate` input instead of a static input file. @@ -178,7 +186,7 @@ You should see logs showing JSON objects similar to the following, with names in kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch ---- + -The pod should now be running without entering a CrashLoopBackOff state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. +The Pod should now be running without entering a `CrashLoopBackOff` state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. === Use streams mode From 4595b09ed82fe692c8e47a3a87cefb02cffa6040 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 7 Nov 2024 11:12:11 +0000 Subject: [PATCH 04/10] Improvements --- .../get-started/pages/quickstarts/helm-chart.adoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 0063bb43..82789de6 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -39,11 +39,11 @@ Redpanda Connect can run in two different modes: **standalone mode** and **strea * **Standalone mode**: Allows you to run a single pipeline at a time, making it suitable for simpler configurations that don't need to run concurrently. * **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing use cases, multiple concurrent data flows. -=== Use standalone mode +=== Run a single pipeline in standalone mode In standalone mode, you configure a single pipeline within the `config` section of your Helm values file. This mode is best suited for running a single pipeline from start to finish in one execution. -In this example, you’ll set up a pipeline that reads data from a file, processes it by converting specific fields to uppercase, and then outputs the result to stdout so you can view it in the logs. +In this example, you'll set up a pipeline that reads data from a file, processes it by converting specific fields to uppercase, and then outputs the result to stdout so you can view it in the logs. . Create a ConfigMap to provide input data that Redpanda Connect will read from a file. This example ConfigMap contains a JSON object with simulated data, such as a user's name and email: + @@ -119,9 +119,9 @@ kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-conn + The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the input file and has no further data to process, the Pod exits, causing Kubernetes to restart it repeatedly. + -To prevent this status, configure Redpanda Connect to continue processing data indefinitely. One way to do this is by using a `generate` input instead of a static input file. +To prevent this status, configure Redpanda Connect to continue processing data indefinitely. One way to do this is by using a `generate` input instead of a static input file. The `generate` input allows Redpanda Connect to continuously produce synthetic data, simulating a live data source. -. Update the `pipeline.yaml` file to use the `generate` input to produce data every second: +. Update the `pipeline.yaml` file to use the `generate` input to produce data every second. + [source,yaml] ---- @@ -188,9 +188,9 @@ kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-conn + The Pod should now be running without entering a `CrashLoopBackOff` state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. -=== Use streams mode +=== Run multiple pipelines in streams mode -Streams mode is designed for running multiple pipelines at once. In streams mode, configurations are managed through a Kubernetes ConfigMap. +Streams mode is suited for concurrent processing and complex workflows. Each pipeline, defined in separate YAML files, runs simultaneously, making this mode ideal for high-throughput applications. All the YAML files must be bundled together into a ConfigMap that you can pass to Redpanda Connect. . Define your pipeline configurations in the following separate YAML files: + From c494aeffcb7293c906fee72dcc21a7746ae4fe9a Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Fri, 8 Nov 2024 16:34:15 +0000 Subject: [PATCH 05/10] Apply feedback from review --- .../pages/quickstarts/helm-chart.adoc | 229 +++++++++++++----- 1 file changed, 175 insertions(+), 54 deletions(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 82789de6..0fd73df6 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -2,9 +2,13 @@ :description: Deploy Redpanda Connect on Kubernetes using Helm. :page-aliases: get-started:helm-chart.adoc -This guide explains how to deploy and configure Redpanda Connect on Kubernetes using the Helm chart. +This guide explains how to deploy and configure Redpanda Connect on Kubernetes using the Helm chart. It covers the available deployment modes and walks you through building your own pipelines. -Redpanda Connect is a powerful stream processor that integrates data across various sources (inputs) and sinks (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it a versatile solution for data pipelines that require high performance and resilience. +== What is Redpanda Connect? + +Redpanda Connect is a powerful stream processor that integrates data across various sources (inputs) and sinks (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it an ideal solution for data pipelines that require high performance and resilience. + +Redpanda Connect includes xref:guides:bloblang/walkthrough.adoc[Bloblang], a flexible mapping language for processing data with built-in functions for transformations, random data generation, and more. These functions allow for easy customization of data pipelines. Common use cases for Redpanda Connect include: @@ -15,7 +19,7 @@ Common use cases for Redpanda Connect include: == Prerequisites * A running Kubernetes cluster. -* `kubectl` CLI and `helm` CLI installed. +* The https://kubernetes.io/docs/tasks/tools/[`kubectl` CLI] and the https://helm.sh/docs/intro/install/[`helm` CLI] installed. == Install with Helm @@ -32,61 +36,46 @@ helm install redpanda-connect redpanda/connect --namespace --create- <2> Updates your local Helm repository cache. <3> Installs Redpanda Connect in the given namespace. You can customize this deployment by configuring values in the Helm chart. -== Run example pipelines +== Deployment modes Redpanda Connect can run in two different modes: **standalone mode** and **streams mode**. * **Standalone mode**: Allows you to run a single pipeline at a time, making it suitable for simpler configurations that don't need to run concurrently. -* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing use cases, multiple concurrent data flows. +* **Streams mode**: Supports multiple pipelines running simultaneously, with each pipeline managed through a ConfigMap. Streams mode is ideal for complex data processing use cases. -=== Run a single pipeline in standalone mode +== Run pipelines in standalone mode -In standalone mode, you configure a single pipeline within the `config` section of your Helm values file. This mode is best suited for running a single pipeline from start to finish in one execution. +In standalone mode, you can configure a single pipeline within the `config` section of your Helm values file. This mode is best suited for running a single pipeline from start to finish in one execution. -In this example, you'll set up a pipeline that reads data from a file, processes it by converting specific fields to uppercase, and then outputs the result to stdout so you can view it in the logs. +=== Hello world -. Create a ConfigMap to provide input data that Redpanda Connect will read from a file. This example ConfigMap contains a JSON object with simulated data, such as a user's name and email: -+ -[source,bash] ----- -kubectl create configmap connect-input --from-literal=input-data='{"name": "Test User", "email": "test.user@example.com"}' --namespace ----- -+ -This ConfigMap will act as the source for the file-based input in Redpanda Connect, allowing the pipeline to read and process this structured JSON data. +In this example, you'll produce a simple message and convert it to uppercase using a Bloblang method. . Create a `pipeline.yaml` file. + This file will be used to override the default Helm chart values for Redpanda Connect. -. Add the following configuration to `pipeline.yaml` to set up a simple standalone pipeline that reads data from the file mounted by the ConfigMap, processes it by converting usernames to uppercase, and outputs the result to stdout: +. Add the following configuration to `pipeline.yaml`: + -.`pipeline.yaml` [source,yaml] ---- -extraVolumes: - - name: input-config - configMap: - name: connect-input -extraVolumeMounts: - - name: input-config - mountPath: /input - subPath: input-data config: input: - file: - paths: - - "/input" <1> + generate: + mapping: | + root.message = "Hello, Redpanda Connect!" <1> + count: 1 <2> pipeline: processors: - mapping: | - root.name = this.name.uppercase() <2> + root.message = this.message.uppercase() <3> output: - stdout: {} <3> + stdout: {} <4> ---- -+ -<1> For the input, use the contents of the file at the path where the ConfigMap data is mounted. -<2> Convert the name field in the input JSON to uppercase using xref:guides:bloblang/walkthrough.adoc[Bloblang], the native mapping language of Redpanda Connect. -<3> Output the processed data to stdout, making it viewable in logs. +<1> Uses Bloblang to create a JSON object with the message "Hello, Redpanda Connect!" stored in `root.message`. +<2> Generate the message only once. +<3> Convert the `message` field in the input to uppercase using the xref:guides:bloblang/methods.adoc#uppercase[`uppercase()`] method in Bloblang. +<4> Output the processed data to stdout, making it viewable in logs. . Deploy the pipeline: + @@ -103,11 +92,11 @@ export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io kubectl logs $POD_NAME --namespace ---- + -You should see the username converted to uppercase in the output: +You should see the messages converted to uppercase in the output: + [,json] ---- -{"name":"TEST USER"} +{"message":"HELLO, REDPANDA CONNECT!"} ---- . Check the Pod's status: @@ -117,11 +106,15 @@ You should see the username converted to uppercase in the output: kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch ---- + -The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the input file and has no further data to process, the Pod exits, causing Kubernetes to restart it repeatedly. +The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the three messages, the Pod exits, causing Kubernetes to restart it repeatedly. + -To prevent this status, configure Redpanda Connect to continue processing data indefinitely. One way to do this is by using a `generate` input instead of a static input file. The `generate` input allows Redpanda Connect to continuously produce synthetic data, simulating a live data source. +To prevent this status, you can configure Redpanda Connect to continue processing data indefinitely. + +=== Produce continuous data -. Update the `pipeline.yaml` file to use the `generate` input to produce data every second. +To produce data continuously, you can set `input.generate.count` to `0`. + +. Update the `pipeline.yaml` file to produce a message every second, indefinitely: + [source,yaml] ---- @@ -131,11 +124,69 @@ config: interval: 1s count: 0 # Setting count to 0 ensures it generates data indefinitely. mapping: | + root.message = "Hello, Redpanda Connect!" + pipeline: + processors: + - mapping: | + root.message = this.uppercase() + output: + stdout: {} +---- + +. Deploy the updated configuration: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Watch the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace -f +---- ++ +You should see in the logs that Redpanda Connect is producing the same message every second and its being converted to uppercase: ++ +[source,json,role="no-copy"] +---- +{"message": "HELLO, REDPANDA CONNECT!"} +{"message": "HELLO, REDPANDA CONNECT!"} +{"message": "HELLO, REDPANDA CONNECT!"} +---- + +. Check the Pod's status: ++ +[source,bash] +---- +kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +---- ++ +The Pod should now be running without entering a `CrashLoopBackOff` state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. + +=== Simulate realistic data streams + +To make the output more realistic, use some Bloblang functions to generate varied data such as random names and emails. + +. Update the `pipeline.yaml` file to generate some realistic user data. ++ +[source,yaml] +---- +config: + input: + generate: + interval: 1s + count: 0 + mapping: | + # Store the generated names in variables let first_name = fake("first_name") let last_name = fake("last_name") + + # Build the message root.user_id = counter() root.name = ($first_name + " " + $last_name) - root.email = ($first_name.slice(0,1) + $last_name + "@example.com") root.timestamp = now() pipeline: processors: @@ -147,10 +198,9 @@ config: + This configuration generates a JSON object with: + -- `user_id`: A unique identifier for each record, generated using the `counter()` function. -- `name`: A randomly generated first and last name, using the `fake()` function. -- `email`: An email based on the generated names. -- `timestamp`: The current timestamp at the time of generation, using the `now()` function. +- `user_id`: A unique identifier for each record, generated using the xref:guides:bloblang/functions.adoc#counter[`counter()`] function. +- `name`: A randomly generated first and last name, using the xref:guides:bloblang/functions.adoc#fake[`fake()`] function. The first and last names are stored in variables and referenced using the `$` syntax. +- `timestamp`: The current timestamp at the time of generation, using the xref:guides:bloblang/functions.adoc#now[`now()`] function. . Deploy the updated configuration: + @@ -179,18 +229,72 @@ You should see logs showing JSON objects similar to the following, with names in {"name":"KALEIGH PACOCHA"} ---- -. Check the Pod's status: +=== Process data from a file input + +To configure a pipeline that reads data from a file, first store the data in a ConfigMap. This ConfigMap will be mounted into the Redpanda Connect Pod, allowing it to read the file directly. + +. Create a ConfigMap to provide the input data that Redpanda Connect will read. This example ConfigMap contains a JSON object with example user data: + [source,bash] ---- -kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch +kubectl create configmap connect-input --from-literal=input-data='{"name": "Redpanda Connect", "email": "rp.connect@example.com"}' --namespace ---- + -The Pod should now be running without entering a `CrashLoopBackOff` state, as the `generate` input continuously feeds new data to the pipeline, preventing it from terminating. +This ConfigMap will act as the source for the file-based input in Redpanda Connect, allowing the pipeline to read and process this structured JSON data. -=== Run multiple pipelines in streams mode +. Update the `pipeline.yaml` file to read data from the file mounted by the ConfigMap: ++ +.`pipeline.yaml` +[source,yaml] +---- +extraVolumes: + - name: input-config + configMap: + name: connect-input +extraVolumeMounts: + - name: input-config + mountPath: /input <1> + subPath: input-data +config: + input: + file: + paths: + - "/input" <1> + pipeline: + processors: + - mapping: | + root.name = this.name.uppercase() + output: + stdout: {} +---- ++ +<1> For the input, use the contents of the file at the path where the ConfigMap data is mounted. -Streams mode is suited for concurrent processing and complex workflows. Each pipeline, defined in separate YAML files, runs simultaneously, making this mode ideal for high-throughput applications. All the YAML files must be bundled together into a ConfigMap that you can pass to Redpanda Connect. +. Deploy the pipeline: ++ +[source,bash] +---- +helm upgrade --install redpanda-connect redpanda/connect --namespace --values pipeline.yaml +---- + +. Check the logs: ++ +[,bash] +---- +export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io/name=redpanda-connect,app.kubernetes.io/instance=redpanda-connect" -o jsonpath="{.items[0].metadata.name}") +kubectl logs $POD_NAME --namespace +---- ++ +You should see the username converted to uppercase in the output: ++ +[,json] +---- +{"name":"REDPANDA CONNECT"} +---- + +== Run multiple pipelines in streams mode + +In streams mode, each pipeline, defined in separate YAML files, runs simultaneously, making this mode ideal for high-throughput applications. All the YAML files must be bundled together into a ConfigMap that you can pass to Redpanda Connect. . Define your pipeline configurations in the following separate YAML files: + @@ -353,6 +457,24 @@ Example output: You can also configure external access using a LoadBalancer Service or an Ingress. See the link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml#L79C1-L107C31[Helm values] for more details. +== Uninstall Redpanda Connect + +To remove Redpanda Connect and all related resources from your Kubernetes cluster, use the following command to uninstall the chart: + +[,bash] +---- +helm uninstall redpanda-connect --namespace +---- + +This command deletes all resources created by the Helm chart, including Deployments and Services. + +Uninstalling the chart does not delete the ConfigMaps that you manually created outside of the Helm chart. To delete these ConfigMaps, do the following: + +[,bash] +---- +kubectl delete configmap connect-streams connect-input --namespace +---- + == Next steps * Learn more about xref:guides:bloblang/walkthrough.adoc[Bloblang], the mapping language for processing data in Redpanda Connect. @@ -360,10 +482,9 @@ You can also configure external access using a LoadBalancer Service or an Ingres == Suggested reading -* link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[Helm values] -* xref:components:http/about.adoc[HTTP server] +* xref:guides:streams_mode/about.adoc[Streams mode] * xref:components:inputs/about.adoc[Inputs] * xref:components:processors/about.adoc[Processors] * xref:components:outputs/about.adoc[Outputs] -* xref:guides:monitoring.adoc[Monitoring] -* xref:configuration:about.adoc[More about configuration] +* xref:components:http/about.adoc[HTTP server] +* link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml[Helm values] From 23bb3b4d35df2f955efd7ef6835d92d6e9e4d1d9 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:50:34 +0000 Subject: [PATCH 06/10] Apply suggestions from code review --- modules/get-started/pages/quickstarts/helm-chart.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 0fd73df6..3e8f62db 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -8,7 +8,7 @@ This guide explains how to deploy and configure Redpanda Connect on Kubernetes u Redpanda Connect is a powerful stream processor that integrates data across various sources (inputs) and sinks (outputs), enabling seamless data flows between systems. It supports complex data processing tasks such as data enrichment, transformation, filtering, and routing, making it an ideal solution for data pipelines that require high performance and resilience. -Redpanda Connect includes xref:guides:bloblang/walkthrough.adoc[Bloblang], a flexible mapping language for processing data with built-in functions for transformations, random data generation, and more. These functions allow for easy customization of data pipelines. +Redpanda Connect includes Bloblang, a flexible mapping language for processing data with built-in functions for transformations, random data generation, and more. These functions allow for easy customization of data pipelines. Common use cases for Redpanda Connect include: From a7d7d80d9463d92618d71ef2de46008f2d5d7fc7 Mon Sep 17 00:00:00 2001 From: Jake Cahill <45230295+JakeSCahill@users.noreply.github.com> Date: Sat, 9 Nov 2024 07:13:59 +0000 Subject: [PATCH 07/10] Update helm-chart.adoc --- modules/get-started/pages/quickstarts/helm-chart.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 3e8f62db..02656c60 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -45,7 +45,7 @@ Redpanda Connect can run in two different modes: **standalone mode** and **strea == Run pipelines in standalone mode -In standalone mode, you can configure a single pipeline within the `config` section of your Helm values file. This mode is best suited for running a single pipeline from start to finish in one execution. +In standalone mode, you can configure a single pipeline within the `config` section of your Helm values file. === Hello world @@ -72,7 +72,7 @@ config: output: stdout: {} <4> ---- -<1> Uses Bloblang to create a JSON object with the message "Hello, Redpanda Connect!" stored in `root.message`. +<1> Use Bloblang to create a JSON object with the message "Hello, Redpanda Connect!" stored in `root.message`. <2> Generate the message only once. <3> Convert the `message` field in the input to uppercase using the xref:guides:bloblang/methods.adoc#uppercase[`uppercase()`] method in Bloblang. <4> Output the processed data to stdout, making it viewable in logs. @@ -92,7 +92,7 @@ export POD_NAME=$(kubectl get pods --namespace -l "app.kubernetes.io kubectl logs $POD_NAME --namespace ---- + -You should see the messages converted to uppercase in the output: +You should see the message converted to uppercase in the output: + [,json] ---- @@ -106,7 +106,7 @@ You should see the messages converted to uppercase in the output: kubectl get pods --namespace -l app.kubernetes.io/name=redpanda-connect --watch ---- + -The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the three messages, the Pod exits, causing Kubernetes to restart it repeatedly. +The Pod enters a `CrashLoopBackOff` state because containers are expected to run continuously. When Redpanda Connect finishes processing the pipeline, the Pod exits, causing Kubernetes to restart it repeatedly. + To prevent this status, you can configure Redpanda Connect to continue processing data indefinitely. From 3e6fe727ac6e03516c09c4e03adaca01078fec28 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Tue, 12 Nov 2024 10:21:19 +0000 Subject: [PATCH 08/10] Add to what's new --- modules/get-started/pages/whats-new.adoc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/get-started/pages/whats-new.adoc b/modules/get-started/pages/whats-new.adoc index f96c90f0..33d9a18f 100644 --- a/modules/get-started/pages/whats-new.adoc +++ b/modules/get-started/pages/whats-new.adoc @@ -10,6 +10,14 @@ For a full list of product updates, and details of updates prior to version 4.29 This update includes the beta release of two new outputs for Cloud and Self-Managed, and the introduction of a xref:configuration:telemetry.adoc[telemetry service], which collects and sends anonymized usage statistics to Redpanda. +=== New tooling + +A new Helm chart for deploying Redpanda Connect in Kubernetes is now available. This chart simplifies the deployment and configuration of Redpanda Connect, allowing you to manage and scale Redpanda Connect instances in Kubernetes environments. + +To get started with the Helm chart, see xref:get-started:quickstarts/helm-chart.adoc[]. + +Or, see the release notes for the chart on https://github.com/redpanda-data/helm-charts/releases/tag/connect-3.0.0[GitHub]. + === New outputs (beta) * xref:components:outputs/azure_data_lake_gen2.adoc[`azure_data_lake_gen2`] (certified) From fa3663d3a279f95f0a3ed9ae25b6cc23dafed8c2 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 14 Nov 2024 10:46:31 +0000 Subject: [PATCH 09/10] Add an upgrade section --- .../pages/quickstarts/helm-chart.adoc | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index 02656c60..ae25b28a 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -457,9 +457,47 @@ Example output: You can also configure external access using a LoadBalancer Service or an Ingress. See the link:https://github.com/redpanda-data/helm-charts/blob/main/charts/connect/values.yaml#L79C1-L107C31[Helm values] for more details. +== Upgrade Redpanda Connect + +To upgrade Redpanda Connect to a newer chart version: + +. Ensure your Helm repository is updated to fetch the latest version of the Redpanda chart: ++ +[,bash] +---- +helm repo update +---- + +. Upgrade your existing deployment using the https://helm.sh/docs/helm/helm_upgrade/[`helm upgrade`] command: ++ +[,bash] +---- +helm upgrade redpanda-connect redpanda/connect --namespace --values +---- ++ +Replace with the path to your custom configuration file. + +=== Roll back an upgrade + +If an upgrade introduces issues, you can roll back to a previous release. Helm retains previous deployments, allowing you to roll back with the https://helm.sh/docs/helm/helm_rollback/[`helm rollback`] command: + +[,bash] +---- +helm rollback redpanda-connect --namespace +---- + +To find the revision number, use the https://helm.sh/docs/helm/helm_history/[`helm history`] command: + +[,bash] +---- +helm history redpanda-connect --namespace +---- + +This command lists previous deployments and their revision numbers. + == Uninstall Redpanda Connect -To remove Redpanda Connect and all related resources from your Kubernetes cluster, use the following command to uninstall the chart: +To remove Redpanda Connect and all related resources from your Kubernetes cluster, use the https://helm.sh/docs/helm/helm_uninstall/[`helm uninstall`] command to uninstall the chart: [,bash] ---- From 7117c382f6144e39c153162098c1e6ebdc50ebe7 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Thu, 14 Nov 2024 10:50:51 +0000 Subject: [PATCH 10/10] Fix format --- modules/get-started/pages/quickstarts/helm-chart.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/get-started/pages/quickstarts/helm-chart.adoc b/modules/get-started/pages/quickstarts/helm-chart.adoc index ae25b28a..03920d4b 100644 --- a/modules/get-started/pages/quickstarts/helm-chart.adoc +++ b/modules/get-started/pages/quickstarts/helm-chart.adoc @@ -472,10 +472,10 @@ helm repo update + [,bash] ---- -helm upgrade redpanda-connect redpanda/connect --namespace --values +helm upgrade redpanda-connect redpanda/connect --namespace --values ---- + -Replace with the path to your custom configuration file. +Replace `` with the path to your custom configuration file. === Roll back an upgrade