Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Built-in Pulsar source #2237

Merged
merged 21 commits into from
Dec 3, 2024
Merged

feat: Built-in Pulsar source #2237

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
2586b28
feat: pulsar integration
vigith Nov 22, 2024
d934976
feat: Pulsar source implementation
BulkBeing Nov 22, 2024
0c7062a
Avoid serializing/deserializing pulsar offset
BulkBeing Nov 24, 2024
2d21a59
Conversion from Pulsar message to numaflow message
BulkBeing Nov 24, 2024
1e13ea8
Error handling
BulkBeing Nov 25, 2024
17d4871
Code generation for Pulsar config object
BulkBeing Nov 25, 2024
6dd6a1e
TTested built-in Pulsar source with Pipeline and Monovertex
BulkBeing Nov 25, 2024
c428d90
Formatting, Upgrade Rust version to 1.82
BulkBeing Nov 25, 2024
1d5ee07
Unit test for Pulsar config parsing
BulkBeing Nov 25, 2024
adfc09b
Use rustls instead of native tls
BulkBeing Nov 27, 2024
230ad2b
Code re-arrangement
BulkBeing Nov 28, 2024
a14cd73
Option to allow self-signed certs
BulkBeing Nov 28, 2024
0db2f48
Tests for Pulsar source
BulkBeing Nov 28, 2024
5d210ab
Use docker to run Pulsar test container
BulkBeing Nov 28, 2024
cd0a8a7
Revert to rustls instead of native_tls (Pulsar dependency)
BulkBeing Nov 29, 2024
68057be
Ability to authenticate with Pulsar using JWT Token
BulkBeing Nov 29, 2024
9b9c30b
Code generation
BulkBeing Nov 29, 2024
4ddecbb
Minor refactoring based on PR review
BulkBeing Dec 2, 2024
3d6c5e8
User Documentation for Pulsar Source
BulkBeing Dec 2, 2024
5555524
User doc updates
BulkBeing Dec 2, 2024
8d4d4e1
chore: add version
vigith Dec 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,11 @@ jobs:
- 4222:4222
env:
NATS_EXTRA_ARGS: -js

steps:
- name: Start Pulsar standalone container
run: docker run -d -p 6650:6650 -p 8080:8080 apachepulsar/pulsar:4.0.0 bin/pulsar standalone

- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ RUN chmod +x /bin/numaflow-rs
####################################################################################################
# Rust binary
####################################################################################################
FROM lukemathwalker/cargo-chef:latest-rust-1.81 AS chef
FROM lukemathwalker/cargo-chef:latest-rust-1.82 AS chef
ARG TARGETPLATFORM
WORKDIR /numaflow
RUN apt-get update && apt-get install -y protobuf-compiler
Expand Down
45 changes: 45 additions & 0 deletions api/json-schema/schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21651,6 +21651,48 @@
},
"type": "object"
},
"io.numaproj.numaflow.v1alpha1.PulsarAuth": {
"description": "PulsarAuth defines how to authenticate with Pulsar",
"properties": {
"token": {
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector",
"description": "JWT Token auth"
}
},
"type": "object"
},
"io.numaproj.numaflow.v1alpha1.PulsarSource": {
"properties": {
"auth": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PulsarAuth",
"description": "Auth information"
},
"consumerName": {
"type": "string"
},
"maxUnack": {
"description": "Maximum number of messages that are in not yet acked state. Once this limit is crossed, futher read requests will return empty list.",
"format": "int64",
"type": "integer"
},
"serverAddr": {
"type": "string"
},
"subscriptionName": {
"type": "string"
},
"topic": {
"type": "string"
}
},
"required": [
"serverAddr",
"topic",
"consumerName",
"subscriptionName"
],
"type": "object"
},
"io.numaproj.numaflow.v1alpha1.RedisBufferService": {
"properties": {
"external": {
Expand Down Expand Up @@ -22096,6 +22138,9 @@
"nats": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.NatsSource"
},
"pulsar": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PulsarSource"
},
"serving": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.ServingSource"
},
Expand Down
45 changes: 45 additions & 0 deletions api/openapi-spec/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21637,6 +21637,48 @@
}
}
},
"io.numaproj.numaflow.v1alpha1.PulsarAuth": {
"description": "PulsarAuth defines how to authenticate with Pulsar",
"type": "object",
"properties": {
"token": {
"description": "JWT Token auth",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
}
}
},
"io.numaproj.numaflow.v1alpha1.PulsarSource": {
"type": "object",
"required": [
"serverAddr",
"topic",
"consumerName",
"subscriptionName"
],
"properties": {
"auth": {
"description": "Auth information",
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PulsarAuth"
},
"consumerName": {
"type": "string"
},
"maxUnack": {
"description": "Maximum number of messages that are in not yet acked state. Once this limit is crossed, futher read requests will return empty list.",
"type": "integer",
"format": "int64"
},
"serverAddr": {
"type": "string"
},
"subscriptionName": {
"type": "string"
},
"topic": {
"type": "string"
}
}
},
"io.numaproj.numaflow.v1alpha1.RedisBufferService": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -22083,6 +22125,9 @@
"nats": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.NatsSource"
},
"pulsar": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.PulsarSource"
},
"serving": {
"$ref": "#/definitions/io.numaproj.numaflow.v1alpha1.ServingSource"
},
Expand Down
35 changes: 35 additions & 0 deletions config/base/crds/full/numaflow.numaproj.io_monovertices.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4992,6 +4992,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down
35 changes: 35 additions & 0 deletions config/base/crds/full/numaflow.numaproj.io_pipelines.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9676,6 +9676,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down
35 changes: 35 additions & 0 deletions config/base/crds/full/numaflow.numaproj.io_vertices.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4460,6 +4460,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down
105 changes: 105 additions & 0 deletions config/install.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8179,6 +8179,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down Expand Up @@ -19414,6 +19449,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down Expand Up @@ -25824,6 +25894,41 @@ spec:
- subject
- url
type: object
pulsar:
properties:
auth:
properties:
token:
properties:
key:
type: string
name:
default: ""
type: string
optional:
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
consumerName:
type: string
maxUnack:
format: int32
type: integer
serverAddr:
type: string
subscriptionName:
type: string
topic:
type: string
required:
- consumerName
- serverAddr
- subscriptionName
- topic
type: object
serving:
properties:
auth:
Expand Down
Loading
Loading