Update Fluent Bit image with newest changes

.gitignore

@@ -1 +1,2 @@
 bin
+integ/out

CHANGELOG.md

@@ -0,0 +1 @@
+# Changelog

Dockerfile

@@ -83,22 +83,26 @@ RUN yum upgrade -y \
           nc
 
 COPY --from=builder /fluent-bit /fluent-bit
-COPY --from=aws-fluent-bit-plugins:latest /go/src/github.com/aws/amazon-kinesis-firehose-for-fluent-bit/bin/firehose.so /fluent-bit/firehose.so
-COPY --from=aws-fluent-bit-plugins:latest /go/src/github.com/aws/amazon-cloudwatch-logs-for-fluent-bit/bin/cloudwatch.so /fluent-bit/cloudwatch.so
+COPY --from=aws-fluent-bit-plugins:latest /kinesis-streams/bin/kinesis.so /fluent-bit/kinesis.so
+COPY --from=aws-fluent-bit-plugins:latest /kinesis-firehose/bin/firehose.so /fluent-bit/firehose.so
+COPY --from=aws-fluent-bit-plugins:latest /cloudwatch/bin/cloudwatch.so /fluent-bit/cloudwatch.so
 RUN mkdir -p /fluent-bit/licenses/fluent-bit
 RUN mkdir -p /fluent-bit/licenses/firehose
 RUN mkdir -p /fluent-bit/licenses/cloudwatch
+RUN mkdir -p /fluent-bit/licenses/kinesis
 COPY THIRD-PARTY /fluent-bit/licenses/fluent-bit/
-COPY --from=aws-fluent-bit-plugins:latest /go/src/github.com/aws/amazon-kinesis-firehose-for-fluent-bit/THIRD-PARTY \
-    /go/src/github.com/aws/amazon-kinesis-firehose-for-fluent-bit/LICENSE \
+COPY --from=aws-fluent-bit-plugins:latest /kinesis-firehose/THIRD-PARTY \
+    /kinesis-firehose/LICENSE \
     /fluent-bit/licenses/firehose/
-COPY --from=aws-fluent-bit-plugins:latest /go/src/github.com/aws/amazon-cloudwatch-logs-for-fluent-bit/THIRD-PARTY \
-    /go/src/github.com/aws/amazon-cloudwatch-logs-for-fluent-bit/LICENSE \
+COPY --from=aws-fluent-bit-plugins:latest /cloudwatch/THIRD-PARTY \
+    /cloudwatch/LICENSE \
     /fluent-bit/licenses/cloudwatch/
-
+COPY --from=aws-fluent-bit-plugins:latest /kinesis-streams/THIRD-PARTY \
+    /kinesis-streams/LICENSE \
+    /fluent-bit/licenses/kinesis/
 
 # Optional Metrics endpoint
 EXPOSE 2020
 
 # Entry point
-CMD ["/fluent-bit/bin/fluent-bit", "-e", "/fluent-bit/firehose.so", "-e", "/fluent-bit/cloudwatch.so", "-c", "/fluent-bit/etc/fluent-bit.conf"]
+CMD ["/fluent-bit/bin/fluent-bit", "-e", "/fluent-bit/firehose.so", "-e", "/fluent-bit/cloudwatch.so", "-e", "/fluent-bit/kinesis.so", "-c", "/fluent-bit/etc/fluent-bit.conf"]

Dockerfile.plugins

@@ -1,13 +1,22 @@
 FROM golang:1.12
+ENV GO111MODULE on
 
-ENV GO111MODULE=on
+# Kinesis Streams
+ARG KINESIS_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-streams-for-fluent-bit.git
+ARG KINESIS_PLUGIN_BRANCH=master
+
+RUN git clone $KINESIS_PLUGIN_CLONE_URL /kinesis-streams
+WORKDIR /kinesis-streams
+RUN git fetch && git checkout $KINESIS_PLUGIN_BRANCH
+RUN go mod download
+RUN make release
 
 # Firehose
 ARG FIREHOSE_PLUGIN_CLONE_URL=https://github.com/aws/amazon-kinesis-firehose-for-fluent-bit.git
 ARG FIREHOSE_PLUGIN_BRANCH=master
 
-RUN git clone $FIREHOSE_PLUGIN_CLONE_URL /go/src/github.com/aws/amazon-kinesis-firehose-for-fluent-bit
-WORKDIR /go/src/github.com/aws/amazon-kinesis-firehose-for-fluent-bit
+RUN git clone $FIREHOSE_PLUGIN_CLONE_URL /kinesis-firehose
+WORKDIR /kinesis-firehose
 RUN git fetch && git checkout $FIREHOSE_PLUGIN_BRANCH
 RUN go mod download
 RUN make release
@@ -16,8 +25,8 @@ RUN make release
 ARG CLOUDWATCH_PLUGIN_CLONE_URL=https://github.com/aws/amazon-cloudwatch-logs-for-fluent-bit.git
 ARG CLOUDWATCH_PLUGIN_BRANCH=master
 
-RUN git clone $CLOUDWATCH_PLUGIN_CLONE_URL /go/src/github.com/aws/amazon-cloudwatch-logs-for-fluent-bit
-WORKDIR /go/src/github.com/aws/amazon-cloudwatch-logs-for-fluent-bit
+RUN git clone $CLOUDWATCH_PLUGIN_CLONE_URL /cloudwatch
+WORKDIR /cloudwatch
 RUN git fetch && git checkout $CLOUDWATCH_PLUGIN_BRANCH
 RUN go mod download
 RUN make release

Makefile

@@ -34,9 +34,13 @@ firehose-dev:
 	--no-cache -t aws-fluent-bit-plugins:latest -f Dockerfile.plugins .
 	docker build -t amazon/aws-for-fluent-bit:latest -f Dockerfile .
 
-.PHONY: integ
-integ: release
-	./integ/integ.sh cloudwatch
+.PHONY: kinesis-dev
+kinesis-dev:
+	docker build \
+	--build-arg KINESIS_PLUGIN_CLONE_URL=${KINESIS_PLUGIN_CLONE_URL} \
+	--build-arg KINESIS_PLUGIN_BRANCH=${KINESIS_PLUGIN_BRANCH} \
+	--no-cache -t aws-fluent-bit-plugins:latest -f Dockerfile.plugins .
+	docker build -t amazon/aws-for-fluent-bit:latest -f Dockerfile .
 
 .PHONY: integ-cloudwatch
 integ-cloudwatch: release
@@ -46,10 +50,31 @@ integ-cloudwatch: release
 integ-cloudwatch-dev: cloudwatch-dev
 	./integ/integ.sh cloudwatch
 
-.PHONY: integ-clean
-integ-clean:
-	./integ/integ.sh clean
-
 .PHONY: integ-clean-cloudwatch
 integ-clean-cloudwatch:
 	./integ/integ.sh clean-cloudwatch
+
+.PHONY: integ-kinesis
+integ-kinesis: release
+	./integ/integ.sh kinesis
+
+.PHONY: integ-kinesis-dev
+integ-kinesis-dev: kinesis-dev
+	./integ/integ.sh kinesis
+
+.PHONY: integ-clean-s3
+integ-clean-s3:
+	./integ/integ.sh clean-s3
+
+.PHONY: integ-dev
+integ-dev: release 
+	./integ/integ.sh kinesis
+	./integ/integ.sh cloudwatch
+
+.PHONY: integ
+integ: 
+	./integ/integ.sh cicd
+
+.PHONY: delete-resources
+delete-resources: 
+	./integ/integ.sh delete

README.md

@@ -12,8 +12,8 @@ There are image tags for `latest` and the version of Fluent Bit that is built in
 
 We also provide images in Amazon ECR in each AWS Region for high availability.
 
-| Region         | Registry ID  | Full Image URI                                                          |
-|----------------|--------------|-------------------------------------------------------------------------|
+| Region         | Registry ID  | Full Image URI                                                              |
+|----------------|--------------|-----------------------------------------------------------------------------|
 | us-east-1      | 906394416424 | 906394416424.dkr.ecr.us-east-1.amazonaws.com/aws-for-fluent-bit:latest      |
 | eu-west-1      | 906394416424 | 906394416424.dkr.ecr.eu-west-1.amazonaws.com/aws-for-fluent-bit:latest      |
 | us-west-1      | 906394416424 | 906394416424.dkr.ecr.us-west-1.amazonaws.com/aws-for-fluent-bit:latest      |
@@ -41,11 +41,17 @@ We also provide images in Amazon ECR in each AWS Region for high availability.
 
 #### Releasing
 
-Use `make release` to build the image. To run the integration tests, run `make integ`.
+Use `make release` to build the image. 
+To run the integration tests, run `make integ-dev`. The `make integ-dev` command will run the integration tests for all of our plugins- 
+kinesis streams, and cloudwatch.
+
+To run integration tests separately, execute `make integ-cloudwatch` or `make integ-kinesis`.
 
 #### Developing Features in the AWS Plugins
 
-You can build a version of the image with code in your GitHub fork.
+You can build a version of the image with code in your GitHub fork. To do so, you must need to set the following environment variables.
+Otherwise, you will see an error message like the following one:
+`fatal: repository '/kinesis-streams' or '/cloudwatch' does not exist.`
 
 Set the following environment variables for CloudWatch:
 
@@ -54,15 +60,15 @@ export CLOUDWATCH_PLUGIN_CLONE_URL="Your GitHub fork clone URL"
 export CLOUDWATCH_PLUGIN_BRANCH="Your branch on your fork"
 ```
 
-Or for Firehose:
+Or for Kinesis Streams:
 ```
-export CLOUDWATCH_PLUGIN_CLONE_URL="Your GitHub fork clone URL"
-export CLOUDWATCH_PLUGIN_BRANCH="Your branch on your fork"
+export KINESIS_PLUGIN_CLONE_URL="Your GitHub fork clone URL"
+export KINESIS_PLUGIN_BRANCH="Your branch on your fork"
 ```
 
-Then run `make cloudwatch-dev` or `make firehose-dev` to build the image with your changes.
-
-To run the integration tests on your code, run `make integ-cloudwatch-dev` or `make integ-firehose-dev`.
+Then run `make cloudwatch-dev` or `make kinesis-dev` to build the image with your changes.
+ 
+To run the integration tests on your code, execute `make integ-cloudwatch-dev` or `make integ-kinesis-dev`.
 
 ## License
 

integ/integ.sh

@@ -1,5 +1,4 @@
 export AWS_REGION="us-west-2"
-
 export PROJECT_ROOT="$(pwd)"
 
 test_cloudwatch() {
@@ -20,15 +19,70 @@ clean_cloudwatch() {
 	docker-compose --file ./integ/test_cloudwatch/docker-compose.clean.yml up --abort-on-container-exit
 }
 
+validate_or_clean_s3() {
+	# Validate: validates that appropirate log data is present in the s3 bucket
+	# Clean: deletes all the objects from s3 bucket that were created in the test
+	export S3_ACTION="${1}"
+	docker-compose --file ./integ/test_kinesis/docker-compose.validate-and-clean-s3.yml build
+	docker-compose --file ./integ/test_kinesis/docker-compose.validate-and-clean-s3.yml up --abort-on-container-exit
+}
+
+test_kinesis() {
+	# Generates log data which will be stored on the s3 bucket
+	docker-compose --file ./integ/test_kinesis/docker-compose.test.yml build
+	docker-compose --file ./integ/test_kinesis/docker-compose.test.yml up --abort-on-container-exit
+
+	sleep 120
+	# Creates a file as a flag for the validation failure
+	mkdir -p ./integ/out
+	touch ./integ/out/kinesis-test
+
+	validate_or_clean_s3 validate
+
+	if [ -f ./integ/out/kinesis-test ]; then
+		# if the file still exists, test failed
+		echo "Test failed for kinesis stream."
+		exit 1
+	fi
+}
+
+clean_s3() {
+	validate_or_clean_s3 clean
+}
+
 if [ "${1}" = "cloudwatch" ]; then
+	mkdir -p ./integ/out
+	touch ./integ/out/cloudwatch-test
 	test_cloudwatch
 	clean_cloudwatch
+	if [ -f ./integ/out/cloudwatch-test ]; then
+		# if the file still exists, test failed
+		echo "Test Failed."
+		exit 1
+	fi
 fi
 
 if [ "${1}" = "clean-cloudwatch" ]; then
 	clean_cloudwatch
 fi
 
-if [ "${1}" = "clean" ]; then
-	clean_cloudwatch
+if [ "${1}" = "kinesis" ]; then
+	source ./integ/resources/create_test_resources.sh
+	source ./integ/resources/setup_test_environment.sh
+
+	clean_s3 && test_kinesis
+fi
+
+if [ "${1}" = "clean-s3" ]; then
+	source ./integ/resources/setup_test_environment.sh
+	clean_s3
+fi
+
+if [ "${1}" = "cicd" ]; then
+	source ./integ/resources/setup_test_environment.sh
+	test_cloudwatch && test_kinesis
+fi
+
+if [ "${1}" = "delete" ]; then
+	source ./integ/resources/delete_test_resources.sh
 fi

integ/logger/logscript.sh

@@ -10,4 +10,5 @@ do
 done
 
 sleep 20
+
 exit 0

integ/resources/cfn-kinesis-s3-firehose.yml

@@ -0,0 +1,89 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Stack for Firehose Delivery Stream with Kinesis Data Stream and S3 Bucket.
+Resources:
+  firehoseDeliveryStream:
+    DependsOn:
+      - deliveryStreamPolicy
+    Type: AWS::KinesisFirehose::DeliveryStream
+    Properties:
+      DeliveryStreamType: KinesisStreamAsSource
+      KinesisStreamSourceConfiguration: 
+        KinesisStreamARN: !GetAtt kinesisStream.Arn
+        RoleARN: !GetAtt deliveryStreamRole.Arn
+      ExtendedS3DestinationConfiguration:
+        BucketARN: !Join 
+          - ''
+          - - 'arn:aws:s3:::'
+            - !Ref s3Bucket
+        BufferingHints:
+          IntervalInSeconds: '60'
+          SizeInMBs: '50'
+        CompressionFormat: UNCOMPRESSED
+        Prefix: firehose/
+        RoleARN: !GetAtt deliveryStreamRole.Arn
+  kinesisStream:
+    Type: AWS::Kinesis::Stream
+    Properties:
+      ShardCount: 5
+  s3Bucket:
+    Type: AWS::S3::Bucket
+  deliveryStreamRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Sid: ''
+            Effect: Allow
+            Principal:
+              Service: firehose.amazonaws.com
+            Action: 'sts:AssumeRole'
+            Condition:
+              StringEquals:
+                'sts:ExternalId': !Ref 'AWS::AccountId'
+  deliveryStreamPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: firehose_delivery_stream_policy
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - 's3:AbortMultipartUpload'
+              - 's3:GetBucketLocation'
+              - 's3:GetObject'
+              - 's3:ListBucket'
+              - 's3:ListBucketMultipartUploads'
+              - 's3:PutObject'
+            Resource:
+              - !Join 
+                - ''
+                - - 'arn:aws:s3:::'
+                  - !Ref s3Bucket
+              - !Join 
+                - ''
+                - - 'arn:aws:s3:::'
+                  - !Ref s3Bucket
+                  - '*'
+          - Effect: Allow
+            Action:
+              - 'kinesis:DescribeStream'
+              - 'kinesis:PutRecord'
+              - 'kinesis:PutRecords'
+              - 'kinesis:GetShardIterator'
+              - 'kinesis:GetRecords'
+              - 'kinesis:ListShards'
+              - 'kinesis:DescribeStreamSummary'
+              - 'kinesis:RegisterStreamConsumer'
+            Resource:
+              - !GetAtt kinesisStream.Arn
+      Roles:
+        - !Ref deliveryStreamRole        
+Outputs: 
+  kinesisStream:
+    Description: The name of the kinesis data stream
+    Value: !Ref kinesisStream
+  s3BucketName:
+    Description: The name of the s3 bucket
+    Value: !Ref s3Bucket

integ/resources/create_test_resources.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Deploys the CloudFormation template to create the stack and necessary resources- kinesis data stream, s3 bucket, and kinesis firehose delivery stream
+# The stack name would be- "integ-test-kinesis"
+# Resource (stream, s3, delivery stream) names will start with the stack name "integ-test-fluent-bit"
+aws cloudformation deploy --template-file ./integ/resources/cfn-kinesis-s3-firehose.yml --stack-name integ-test-fluent-bit --region us-west-2 --capabilities CAPABILITY_NAMED_IAM

integ/resources/delete_test_resources.sh

@@ -0,0 +1,2 @@
+# Delete the CloudFormation stack which created all the resources for running the integration test
+aws cloudformation delete-stack --stack-name integ-test-fluent-bit

integ/resources/setup_test_environment.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Using CloudFormation describe-stacks extracts the output values for kinesis stream and s3 bucket name, and sets them as environment variables
+stackOutputs=$(aws cloudformation describe-stacks --stack-name integ-test-fluent-bit --output text --query 'Stacks[0].Outputs[*].OutputValue')
+read -r -a outputArray <<< "$stackOutputs"
+echo "Kinesis Stream: ${outputArray[0]}"
+echo "S3 Bucket Name: ${outputArray[1]}"
+export KINESIS_STREAM="${outputArray[0]}"
+export S3_BUCKET_NAME="${outputArray[1]}"