diff --git a/.github/workflows/actions/setup-keda/actions.yml b/.github/workflows/actions/setup-keda/actions.yml new file mode 100644 index 00000000..46ebcbc2 --- /dev/null +++ b/.github/workflows/actions/setup-keda/actions.yml @@ -0,0 +1,43 @@ +name: Keda Setup + +description: Setup Kind and Keda + +env: + HELM_VERSION: v3.13.0 + KIND_VERSION: v0.20.0 + KIND_CLUSTER_NAME: kind + KEDA_VERSION: 2.12.0 + +runs: + using: "composite" + steps: + - name: Check out code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup Kind cluster + uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 + with: + node_image: kindest/node:v1.23.17 + version: ${{ env.KIND_VERSION }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + + - name: Helm installation + uses: Azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + with: + version: ${{ env.HELM_VERSION }} + + - name: Install Keda + run: | + helm repo add kedacore https://kedacore.github.io/charts + helm repo update + helm install keda kedacore/keda --namespace keda --version ${{ env.KEDA_VERSION }} + + - name: Setup Ko + uses: ko-build/setup-ko@ace48d793556083a76f1e3e6068850c1f4a369aa # v0.6 + env: + KO_DOCKER_REPO: kind.local + + - name: Run local http server + run: | + ko resolve -f test/server/server.yaml | kubectl apply -f - + kubectl wait pod -l app=http --for=condition=ready --timeout=-1s diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96b79034..3e8e26ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,10 @@ jobs: outputs: packages: ${{ steps.filter.outputs.changes }} steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 + - name: Check out code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 id: filter with: filters: | @@ -41,104 +42,59 @@ jobs: if: contains(needs.check.outputs.packages, 'kafkachanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s - - name: Create Docker Image for Kafka KEDA Connector - run: | - cd kafka-http-connector/ - docker build -t localhost:5000/kafka-connector:latest . - docker push localhost:5000/kafka-connector:latest - - name: Create Kafka Docker Image for Consumer - run: | - cd kafka-http-connector/test/consumer/ - docker build -t localhost:5000/consumer:latest . - docker push localhost:5000/consumer:latest - - name: Deploy Kafka cluster + - name: Setup Keda + uses: ./.github/actions/setup-keda + + - name: Deploy Kafka Cluster run: | cd kafka-http-connector/test/kubernetes/ kubectl create namespace kafka curl -L http://strimzi.io/install/latest | sed 's/namespace: .*/namespace: kafka/' | kubectl create -f - -n kafka sleep 10s - kubectl create -f kafka-cluster.yaml + kubectl create -f kafka-http-connector/test/kubernetes/kafka-cluster.yaml echo "Kafka Cluster is getting up." kubectl wait -f kafka-cluster.yaml --for=condition=ready --timeout=-1s -n kafka sleep 2m kubectl get pods -n kafka kubectl wait pod -l app.kubernetes.io/name=zookeeper --for=condition=ready --timeout=-1s -n kafka + - name: Create Kafka topics run: | cd kafka-http-connector/test/kubernetes/ kubectl apply -f kafka-req-topic.yaml kubectl apply -f kafka-err-topic.yaml kubectl apply -f kafka-res-topic.yaml + - name: Deploy Kafka Keda Connector and Keda ScaledObject run: | - cd kafka-http-connector/test/kubernetes/ - kubectl apply -f keda-deployment.yml + ko resolve -f kafka-http-connector/test/kubernetes/keda-deployment.yml | kubectl apply -f - -n kafka kubectl get pods -n kafka - docker images sleep 15s kubectl get pods -n kafka kubectl wait pod -l keda=kafka --for=condition=ready --timeout=30s -n kafka - kubectl apply -f Keda-ScaledObj.yml + kubectl apply -f kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml -n kafka + - name: Produce Kafka messages Using Producer run: | - cd kafka-http-connector/test/kubernetes/ - kubectl apply -f kafka-produer.yaml -n kafka - kubectl wait -f kafka-produer.yaml --for=condition=complete --timeout=-1s -n kafka + ko resolve -f kafka-http-connector/test/kubernetes/kafka-produer.yaml | kubectl apply -f - -n kafka + kubectl wait -l app=pi --for=condition=complete --timeout=-1s -n kafka kubectl delete job pi -n kafka + - name: Collect Kafka Consumer Messages run: | kubectl get pods -n kafka - cd kafka-http-connector/test/consumer/ - kubectl apply -f consumer-deployment.yaml + ko resolve -f kafka-http-connector/test/consumer/consumer-deployment.yaml | kubectl apply -f - -n kafka kubectl wait pod -l app=consumer --for=condition=ready --timeout=-1s -n kafka kubectl logs -l app=consumer --all-containers=true -n kafka + rabbitmq: needs: check if: contains(needs.check.outputs.packages, 'rabbitmqchanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s + - name: Setup Keda + uses: ./.github/actions/setup-keda + - name: Create Docker Image for Rabbitmq KEDA Connector run: | cd rabbitmq-http-connector/ @@ -187,27 +143,9 @@ jobs: if: contains(needs.check.outputs.packages, 'sqschanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s + - name: Setup Keda + uses: ./.github/actions/setup-keda + - name: Create Docker Image for SQS KEDA Connector run: | cd aws-sqs-http-connector @@ -242,27 +180,9 @@ jobs: if: contains(needs.check.outputs.packages, 'redischanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s + - name: Setup Keda + uses: ./.github/actions/setup-keda + - name: Create Docker Image for Redis KEDA Connector run: | cd redis-http-connector/ @@ -316,27 +236,9 @@ jobs: if: contains(needs.check.outputs.packages, 'natschanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s + - name: Setup Keda + uses: ./.github/actions/setup-keda + - name: Create Docker Image for Nats KEDA Connector run: | cd nats-streaming-http-connector/ @@ -389,27 +291,9 @@ jobs: if: contains(needs.check.outputs.packages, 'natsjetstreamchanges') runs-on: ubuntu-latest steps: - - name: Checkout the current repo - uses: actions/checkout@v3 - - name: Install Kind Cluster - uses: engineerd/setup-kind@v0.5.0 - with: - version: v0.18.0 - - name: Install KEDA using HELM - run: | - cd test/ - ./registry.sh - helm repo add kedacore https://kedacore.github.io/charts - helm repo update - kubectl create namespace keda - helm install keda kedacore/keda --namespace keda - - name: Create Docker Image for HTTP server - run: | - cd test/server/ - docker build -t localhost:5000/http-server:latest . - docker push localhost:5000/http-server:latest - kubectl apply -f server.yaml - kubectl wait pod -l app=http --for=condition=ready --timeout=-1s + - name: Setup Keda + uses: ./.github/actions/setup-keda + - name: Create Docker Image for Jetstream Nats KEDA Connector run: | cd nats-jetstream-http-connector/ @@ -428,7 +312,7 @@ jobs: - name: Deploy Jetstream Nats cluster run: | cd nats-jetstream-http-connector/test/jetstream - kubectl apply -f jetstream-server.yaml + kubectl apply -f jetstream-server.yaml - name: Publish Nats messages in a stream run: | cd nats-jetstream-http-connector/test/producer/ @@ -450,4 +334,4 @@ jobs: # sleep 90s kubectl wait pod -l app=consumer --for=condition=ready --timeout=-1s kubectl logs deployment.apps/consumer-deployment | grep -q "consumed" - echo $? \ No newline at end of file + echo $? diff --git a/kafka-http-connector/test/consumer/consumer-deployment.yaml b/kafka-http-connector/test/consumer/consumer-deployment.yaml index 6c093905..be6d3957 100644 --- a/kafka-http-connector/test/consumer/consumer-deployment.yaml +++ b/kafka-http-connector/test/consumer/consumer-deployment.yaml @@ -16,5 +16,5 @@ spec: app: consumer spec: containers: - - name: consumer - image: localhost:5000/consumer:latest \ No newline at end of file + - name: consumer + image: ko://github.com/fission/keda-connectors/kafka-http-connector/test/consumer diff --git a/kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml b/kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml index 656cf13d..0ed52baa 100644 --- a/kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml +++ b/kafka-http-connector/test/kubernetes/Keda-ScaledObj.yml @@ -10,9 +10,8 @@ spec: name: keda-deployment pollingInterval: 30 triggers: - - type: kafka - metadata: - bootstrapServers: my-cluster-test-kafka-bootstrap:9092 - consumerGroup: bridge-quickstart-consumer-group - topic: request-topic - + - type: kafka + metadata: + bootstrapServers: my-cluster-test-kafka-bootstrap:9092 + consumerGroup: bridge-quickstart-consumer-group + topic: request-topic diff --git a/kafka-http-connector/test/kubernetes/kafka-produer.yaml b/kafka-http-connector/test/kubernetes/kafka-produer.yaml index 1b9d3ed5..722f3174 100644 --- a/kafka-http-connector/test/kubernetes/kafka-produer.yaml +++ b/kafka-http-connector/test/kubernetes/kafka-produer.yaml @@ -2,6 +2,8 @@ apiVersion: batch/v1 kind: Job metadata: name: pi + labels: + app: pi spec: completions: 1 ttlSecondsAfterFinished: 10 diff --git a/kafka-http-connector/test/kubernetes/keda-deployment.yml b/kafka-http-connector/test/kubernetes/keda-deployment.yml index 2d138787..a4dc8520 100644 --- a/kafka-http-connector/test/kubernetes/keda-deployment.yml +++ b/kafka-http-connector/test/kubernetes/keda-deployment.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: keda-kafka-connector - image: localhost:5000/kafka-connector:latest + image: ko://github.com/fission/keda-connectors/kafka-http-connector env: - name: TOPIC value: request-topic diff --git a/test/server/server.yaml b/test/server/server.yaml index eda626a5..4c2eb69d 100644 --- a/test/server/server.yaml +++ b/test/server/server.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: http-server - image: localhost:5000/http-server:latest + image: ko://github.com/fission/keda-connectors/test/server ports: - containerPort: 8080 ---