[libbeat][reader] - Adding support for parquet reader (#35183)

* initial commit for s3 parquet support

* updated changelog

* added license updates

* updated notice and go mod/sum

* removed libgering panic

* added parquet benchmark tests

* updated osquery package due to update in dependant thrift package

* added parquet reader with benchmark tests and implemented that reader in awss3 package

* addressed linting errors

* refactored parquet reader, added tests and benchmarks and addressed pr comments

* addressed pr comments

* resolved merged conflicts

* updated notice

* added more parquet file tests with json comparisons, addressed pr comments

* removed commented codeS

* removed bad imports & cleaned up tests

* updated notice

* added graceful closures with err checks in test

* added graceful closures with err checks in test

* removed s3 parquet implementation from this PR

* removed s3 parquet implementation from this PR

* Update filebeat.yml

* Update filebeat.yml

* updated notice

* addressed PR suggestions

* addressed PR comments

* updated godoc comment

* addressed PR comments, switched path with filebath

* updated CODEOWNERS and addressed PR comments

* addressed PR comments, added a rand seeding process

* fixed test seed value to 1

* updated comments

* removed defers in loops

* updated notice

* updated godoc comments as suggested

* updated changelog

* Update x-pack/libbeat/reader/parquet/parquet.go

Co-authored-by: subham sarkar <[email protected]>

---------

Co-authored-by: subham sarkar <[email protected]>

.github/CODEOWNERS

@@ -154,3 +154,4 @@ CHANGELOG*
 /x-pack/osquerybeat/ @elastic/security-asset-management
 /x-pack/packetbeat/ @elastic/security-external-integrations
 /x-pack/winlogbeat/ @elastic/security-external-integrations
+/x-pack/libbeat/reader/parquet/ @elastic/security-external-integrations

CHANGELOG.next.asciidoc

@@ -291,6 +291,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 
 *Filebeat*
 
+*Libbeat*
+- Added support for apache parquet file reader. {issue}34662[34662] {pull}35183[35183]
 
 *Heartbeat*
 - Users can now configure max scheduler job limits per monitor type via env var. {pull}34307[34307]

dev-tools/notice/overrides.json

@@ -16,3 +16,4 @@
 {"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
 {"name": "github.com/awslabs/kinesis-aggregation/go/v2", "licenceType": "Apache-2.0", "url": "https://github.com/awslabs/kinesis-aggregation/blob/master/LICENSE.txt"}
 {"name": "github.com/dnaeon/go-vcr", "licenceType": "BSD-2-Clause"}
+{"name": "github.com/JohnCGriffin/overflow", "licenceType": "MIT"}

dev-tools/notice/rules.json

@@ -5,6 +5,7 @@
     "BSD-2-Clause",
     "BSD-2-Clause-FreeBSD",
     "BSD-3-Clause",
+    "CC0-1.0",
     "Elastic",
     "ISC",
     "MIT",

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
 	github.com/Azure/go-autorest/autorest/date v0.3.0
 	github.com/Masterminds/semver v1.5.0
-	github.com/Microsoft/go-winio v0.6.0
+	github.com/Microsoft/go-winio v0.6.1
 	github.com/PaesslerAG/gval v1.0.0
 	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/Shopify/sarama v1.27.0
@@ -99,7 +99,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
 	github.com/gomodule/redigo v1.8.3
-	github.com/google/flatbuffers v1.12.1
+	github.com/google/flatbuffers v23.3.3+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/gopacket v1.1.19
 	github.com/google/uuid v1.3.0
@@ -126,7 +126,7 @@ require (
 	github.com/mitchellh/hashstructure v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/osquery/osquery-go v0.0.0-20210622151333-99b4efa62ec5
+	github.com/osquery/osquery-go v0.0.0-20220706183148-4e1f83012b42
 	github.com/otiai10/copy v1.2.0
 	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
@@ -188,6 +188,7 @@ require (
 	cloud.google.com/go/redis v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
 	github.com/Azure/go-autorest/autorest/adal v0.9.14
+	github.com/apache/arrow/go/v11 v11.0.0
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4
 	github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8
@@ -234,8 +235,10 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
+	github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
-	github.com/apache/thrift v0.13.1-0.20200603211036-eac4d0c79a5f // indirect
+	github.com/apache/thrift v0.18.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.38.60 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3 // indirect
@@ -270,6 +273,7 @@ require (
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/gobuffalo/here v0.6.7 // indirect
+	github.com/goccy/go-json v0.9.11 // indirect
 	github.com/godror/knownpb v0.1.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
@@ -298,13 +302,17 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/karrick/godirwalk v1.17.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/markbates/pkger v0.17.1 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/iochan v1.0.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
@@ -333,6 +341,7 @@ require (
 	github.com/xdg/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect