Merge pull request #11 from zinal/features1

BulkUpsert request size limit and `.others` column for extra data as JSON
ydb-platform · May 2, 2024 · 078e9e9 · 078e9e9
2 parents a86b559 + f7790af
commit 078e9e9
Show file tree

Hide file tree

Showing 7 changed files with 274 additions and 41 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,8 @@
+* Added the logic to automatically limit the request sizes for `BulkUpsert`, to avoid the ingestion errors
+* Added the saving extra input fields as the (optional) additional JSON-formatted field named `.other`
+* Supported the flexible schema parsers like `logfmt` on input
+* Fixed the lost of same-time messages by (optionally) adding extra `.hash` field containing the `Cityhash64` computed over the record
+
 ## v1.1.1
 * Fixed Dockerfile for build with go1.22
 

diff --git a/README.md b/README.md
@@ -15,8 +15,7 @@
 ## Build
 
 ```makefile
-export BIN=out_ydb.so
-make build
+BIN=out_ydb.so make build
 ```
 
 # Usage 

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/fluent/fluent-bit-go v0.0.0-20230731091245-a7a013e2473c
 	github.com/stretchr/testify v1.8.3
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.66.0
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.66.1
 	github.com/ydb-platform/ydb-go-yc v0.12.1
 )
 
@@ -16,6 +16,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/surge/cityhash v0.0.0-20131128155616-cdd6a94144ab // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20240425114406-68c9b49389a1 // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect

diff --git a/go.sum b/go.sum
@@ -773,6 +773,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/surge/cityhash v0.0.0-20131128155616-cdd6a94144ab h1:Kmv2LOAf1bYObq0HW/XuLP4U92z3aXVHhAgdvE2u7BQ=
+github.com/surge/cityhash v0.0.0-20131128155616-cdd6a94144ab/go.mod h1:o8cYsNqWX8QahvKFMeXIFD1R5+df885pkwh8Vo/htck=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
@@ -790,6 +792,8 @@ github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGg
 github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.66.0 h1:ffZ1Kh60rA62BVHVafH/xrIn5yrlJ/ZDvIoVg7mhVGo=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.66.0/go.mod h1:hGX4CijskNnUTRgLlqMvZdrBQc1ALZgAnKHytF5nmj4=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.66.1 h1:K6TNapbuMFTFFMil26rP3fMgZmuLE9jP8curXRyKlZs=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.66.1/go.mod h1:hGX4CijskNnUTRgLlqMvZdrBQc1ALZgAnKHytF5nmj4=
 github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
 github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
 github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -28,6 +28,8 @@ const (
 
 	KeyTimestamp = ".timestamp"
 	KeyInput     = ".input"
+	KeyOthers    = ".others"
+	KeyHash      = ".hash"
 )
 
 type credentialsDescription struct {