From 4164728ac2be8f2b58391faacb71ed11267db35e Mon Sep 17 00:00:00 2001 From: yuryalekseev Date: Thu, 9 Jan 2025 19:45:02 +0300 Subject: [PATCH 1/5] YT-23542: Address review comments. commit_hash:2a32a33300fcdb3f636d67fe644533246d101281 --- yt/yt/core/concurrency/throughput_throttler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/yt/core/concurrency/throughput_throttler.h b/yt/yt/core/concurrency/throughput_throttler.h index e7b2f8129912..8d271881d890 100644 --- a/yt/yt/core/concurrency/throughput_throttler.h +++ b/yt/yt/core/concurrency/throughput_throttler.h @@ -67,7 +67,7 @@ struct IThroughputThrottler */ virtual bool IsOverdraft() = 0; - //! Returns total byte amount of all waiting requests. + //! Returns total amount of units of all waiting requests. /*! * \note Thread affinity: any */ From 5b95a133cb44f9f6dd65110f057cc55b36f9a859 Mon Sep 17 00:00:00 2001 From: thegeorg Date: Thu, 9 Jan 2025 19:48:50 +0300 Subject: [PATCH 2/5] Cleanup libunwind import routine commit_hash:edc483d8a602a84f9a90fe73a5889591c1d8ac46 --- .../libs/libunwind/.yandex_meta/__init__.py | 4 ++-- .../libs/libunwind/.yandex_meta/override.nix | 5 +++++ .../libunwind/include/__libunwind_config.h | 4 ++++ .../libunwind/patches/04-wasm-config.patch | 19 +++++++++++++++++++ contrib/libs/libunwind/ya.make | 8 ++++++-- 5 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 contrib/libs/libunwind/patches/04-wasm-config.patch diff --git a/contrib/libs/libunwind/.yandex_meta/__init__.py b/contrib/libs/libunwind/.yandex_meta/__init__.py index f93d859c2e19..d1a5b082d631 100644 --- a/contrib/libs/libunwind/.yandex_meta/__init__.py +++ b/contrib/libs/libunwind/.yandex_meta/__init__.py @@ -4,12 +4,12 @@ def post_install(self): with self.yamakes["."] as libunwind: - libunwind.DISABLE.add("USE_LTO") libunwind.NO_COMPILER_WARNINGS = False + libunwind.NO_LTO = True libunwind.NO_RUNTIME = True libunwind.NO_SANITIZE = True libunwind.NO_SANITIZE_COVERAGE = True - libunwind.ADDINCL = [f"{self.arcdir}/include"] + # original build uses -f options heavily, keep only necessary subset libunwind.CFLAGS += ["-fno-exceptions", "-fno-rtti", "-funwind-tables"] libunwind.after("CFLAGS", Switch({"SANITIZER_TYPE == memory": "CFLAGS(-fPIC)"})) libunwind.PEERDIR.add("library/cpp/sanitizer/include") diff --git a/contrib/libs/libunwind/.yandex_meta/override.nix b/contrib/libs/libunwind/.yandex_meta/override.nix index f6fbeb5805da..04e59398653a 100644 --- a/contrib/libs/libunwind/.yandex_meta/override.nix +++ b/contrib/libs/libunwind/.yandex_meta/override.nix @@ -11,4 +11,9 @@ pkgs: attrs: with pkgs; with attrs; rec { patches = []; sourceRoot = "source/libunwind"; + + # Building without this option breaks build traces: https://st.yandex-team.ru/DTCC-589. + cmakeFlags = [ + "-DCMAKE_CXX_FLAGS=-fno-integrated-cc1" + ]; } diff --git a/contrib/libs/libunwind/include/__libunwind_config.h b/contrib/libs/libunwind/include/__libunwind_config.h index 8db336b2d727..028b9e3baa80 100644 --- a/contrib/libs/libunwind/include/__libunwind_config.h +++ b/contrib/libs/libunwind/include/__libunwind_config.h @@ -180,6 +180,10 @@ #endif #define _LIBUNWIND_HIGHEST_DWARF_REGISTER \ _LIBUNWIND_HIGHEST_DWARF_REGISTER_LOONGARCH +#elif defined(__wasm__) +// Unused +#define _LIBUNWIND_CONTEXT_SIZE 0 +#define _LIBUNWIND_CURSOR_SIZE 0 # else # error "Unsupported architecture." # endif diff --git a/contrib/libs/libunwind/patches/04-wasm-config.patch b/contrib/libs/libunwind/patches/04-wasm-config.patch new file mode 100644 index 000000000000..c093f4754155 --- /dev/null +++ b/contrib/libs/libunwind/patches/04-wasm-config.patch @@ -0,0 +1,19 @@ +commit 080afaf1ef8e81cfdc26850560b639acd0c88733 +author: robot-contrib +date: 2024-12-28T08:39:56+03:00 + + Update contrib/libs/libunwind to 19.1.5 + +--- contrib/libs/libunwind/include/__libunwind_config.h (cc7e7e3cd08b4064e3856fcec6df5aa39b8cc0a4) ++++ contrib/libs/libunwind/include/__libunwind_config.h (080afaf1ef8e81cfdc26850560b639acd0c88733) +@@ -180,6 +180,10 @@ + #endif + #define _LIBUNWIND_HIGHEST_DWARF_REGISTER \ + _LIBUNWIND_HIGHEST_DWARF_REGISTER_LOONGARCH ++#elif defined(__wasm__) ++// Unused ++#define _LIBUNWIND_CONTEXT_SIZE 0 ++#define _LIBUNWIND_CURSOR_SIZE 0 + # else + # error "Unsupported architecture." + # endif diff --git a/contrib/libs/libunwind/ya.make b/contrib/libs/libunwind/ya.make index 6556b688ee5a..062d70272972 100644 --- a/contrib/libs/libunwind/ya.make +++ b/contrib/libs/libunwind/ya.make @@ -15,8 +15,6 @@ VERSION(18.1.0-rc1) ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/llvmorg-18.1.0-rc1.tar.gz) -DISABLE(USE_LTO) - PEERDIR( library/cpp/sanitizer/include ) @@ -29,11 +27,17 @@ NO_RUNTIME() NO_UTIL() +NO_LTO() + NO_SANITIZE() NO_SANITIZE_COVERAGE() CFLAGS( + -D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER + -D_LIBUNWIND_IS_NATIVE_ONLY + -D_LIBUNWIND_LINK_DL_LIB + -D_LIBUNWIND_LINK_PTHREAD_LIB -fno-exceptions -fno-rtti -funwind-tables From 7740c9ef0cd954624d51f1fc8c8ff4426edcc9ab Mon Sep 17 00:00:00 2001 From: yuryalekseev Date: Thu, 9 Jan 2025 19:51:56 +0300 Subject: [PATCH 3/5] YT-23938: Prohibit null type in IsCGComparatorApplicable(). commit_hash:49523c81c33742bfb087e1df870e2c548960afe5 --- yt/yt/client/table_client/schema.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index 7416c7462e2c..46cfbee1b23e 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -812,7 +812,7 @@ bool TTableSchema::IsCGComparatorApplicable() const auto keyTypes = GetKeyColumnTypes(); return std::none_of(keyTypes.begin(), keyTypes.end(), [] (auto type) { - return type == EValueType::Any; + return type == EValueType::Any || type == EValueType::Null; }); } From 667707fa06732f2943922d08ef77a2c9a52cbd47 Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Thu, 9 Jan 2025 20:55:43 +0300 Subject: [PATCH 4/5] Intermediate changes commit_hash:c97917e5fe0a2f094b8f7a50cf183357e8d07096 --- .../libs/libunwind/.yandex_meta/__init__.py | 2 +- library/cpp/protobuf/yql/descriptor.cpp | 22 ++++++- library/cpp/protobuf/yql/descriptor.h | 5 ++ .../protobuf/test/canondata/result.json | 5 ++ .../test.test_YQL-19040_/results.txt | 28 +++++++++ .../common/protobuf/test/cases/YQL-19040.sql | 62 +++++++++++++++++++ 6 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 yql/essentials/udfs/common/protobuf/test/canondata/test.test_YQL-19040_/results.txt create mode 100644 yql/essentials/udfs/common/protobuf/test/cases/YQL-19040.sql diff --git a/contrib/libs/libunwind/.yandex_meta/__init__.py b/contrib/libs/libunwind/.yandex_meta/__init__.py index d1a5b082d631..803baf46fbdc 100644 --- a/contrib/libs/libunwind/.yandex_meta/__init__.py +++ b/contrib/libs/libunwind/.yandex_meta/__init__.py @@ -9,7 +9,7 @@ def post_install(self): libunwind.NO_RUNTIME = True libunwind.NO_SANITIZE = True libunwind.NO_SANITIZE_COVERAGE = True - # original build uses -f options heavily, keep only necessary subset + # original build uses -f options heavily, keep only necessary subset libunwind.CFLAGS += ["-fno-exceptions", "-fno-rtti", "-funwind-tables"] libunwind.after("CFLAGS", Switch({"SANITIZER_TYPE == memory": "CFLAGS(-fPIC)"})) libunwind.PEERDIR.add("library/cpp/sanitizer/include") diff --git a/library/cpp/protobuf/yql/descriptor.cpp b/library/cpp/protobuf/yql/descriptor.cpp index 2d5a154fc177..e5e1c8e28bdd 100644 --- a/library/cpp/protobuf/yql/descriptor.cpp +++ b/library/cpp/protobuf/yql/descriptor.cpp @@ -19,6 +19,7 @@ #include #include +#include using namespace NProtoBuf; @@ -73,6 +74,7 @@ TDynamicInfoRef TDynamicInfo::Create(const TStringBuf& typeConfig) { info->SkipBytes_ = data.SkipBytes; info->OptionalLists_ = data.OptionalLists; info->SyntaxAware_ = data.SyntaxAware; + info->Deterministic_ = data.Deterministic; return info; } @@ -146,7 +148,16 @@ TString TDynamicInfo::Serialize(const Message& proto) { switch (ProtoFormat_) { case PF_PROTOBIN: { result.ReserveAndResize(proto.ByteSize()); - if (!proto.SerializeToArray(result.begin(), result.size())) { + bool success = false; + if (Deterministic_) { + io::ArrayOutputStream arrOut(result.begin(), result.size()); + io::CodedOutputStream codedOut(&arrOut); + codedOut.SetSerializationDeterministic(true); + success = proto.SerializeToCodedStream(&codedOut); + } else { + success = proto.SerializeToArray(result.begin(), result.size()); + } + if (!success) { ythrow yexception() << "can't serialize protobin message"; } break; @@ -159,7 +170,9 @@ TString TDynamicInfo::Serialize(const Message& proto) { } case PF_JSON: { NJson::TJsonValue value; - NProtobufJson::Proto2Json(proto, value); + NProtobufJson::TProto2JsonConfig config; + config.SetSortMapKeys(Deterministic_); + NProtobufJson::Proto2Json(proto, value, config); result = NJson::WriteJson(value); break; } @@ -225,6 +238,10 @@ TString GenerateProtobufTypeConfig( ret["view"]["yt_mode"] = true; } + if (options.Deterministic) { + ret["view"]["deterministic"] = true; + } + return NJson::WriteJson(ret, false); } @@ -268,6 +285,7 @@ TProtoTypeConfig ParseTypeConfig(const TStringBuf& config) { result.OptionalLists = value["lists"]["optional"].GetBooleanSafe(true); result.SyntaxAware = value["syntax"]["aware"].GetBooleanSafe(false); result.YtMode = value["view"]["yt_mode"].GetBooleanSafe(false); + result.Deterministic = value["view"]["deterministic"].GetBooleanSafe(false); if (protoFormat == "protobin") { result.ProtoFormat = PF_PROTOBIN; diff --git a/library/cpp/protobuf/yql/descriptor.h b/library/cpp/protobuf/yql/descriptor.h index bbed6850ec10..f5f51add0bcb 100644 --- a/library/cpp/protobuf/yql/descriptor.h +++ b/library/cpp/protobuf/yql/descriptor.h @@ -50,6 +50,8 @@ struct TProtoTypeConfig { bool OptionalLists = false; //! Заполнять ли пустые Optional типы дефолтным значением (только для proto3). bool SyntaxAware = false; + //! Использовать ли детерминированную сериализацию + bool Deterministic = false; }; struct TProtoTypeConfigOptions { @@ -69,6 +71,8 @@ struct TProtoTypeConfigOptions { bool OptionalLists = false; //! Заполнять ли пустые Optional типы дефолтным значением (только для proto3). bool SyntaxAware = false; + //! Использовать ли детерминированную сериализацию + bool Deterministic = false; TProtoTypeConfigOptions& SetProtoFormat(EProtoFormat value) { ProtoFormat = value; @@ -158,4 +162,5 @@ class TDynamicInfo: public TSimpleRefCount { ui32 SkipBytes_; bool OptionalLists_; bool SyntaxAware_; + bool Deterministic_; }; diff --git a/yql/essentials/udfs/common/protobuf/test/canondata/result.json b/yql/essentials/udfs/common/protobuf/test/canondata/result.json index ac534cee58fc..be8251f2ae00 100644 --- a/yql/essentials/udfs/common/protobuf/test/canondata/result.json +++ b/yql/essentials/udfs/common/protobuf/test/canondata/result.json @@ -13,6 +13,11 @@ "uri": "https://storage.yandex-team.ru/get-devtools/1781765/c2a453956382e0fdfc958f3c4e32ed7740f03d4b/resource.tar.gz#test.test_YQL-16111_/results.txt" } ], + "test.test[YQL-19040]": [ + { + "uri": "file://test.test_YQL-19040_/results.txt" + } + ], "test.test[YQL-3381]": [ { "checksum": "c098af301d5dc8d85071a47455f0f592", diff --git a/yql/essentials/udfs/common/protobuf/test/canondata/test.test_YQL-19040_/results.txt b/yql/essentials/udfs/common/protobuf/test/canondata/test.test_YQL-19040_/results.txt new file mode 100644 index 000000000000..67f6ba046e3f --- /dev/null +++ b/yql/essentials/udfs/common/protobuf/test/canondata/test.test_YQL-19040_/results.txt @@ -0,0 +1,28 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "DataType"; + "String" + ] + ] + ] + ] + ]; + "Data" = [ + [ + "Success" + ] + ] + } + ] + } +] \ No newline at end of file diff --git a/yql/essentials/udfs/common/protobuf/test/cases/YQL-19040.sql b/yql/essentials/udfs/common/protobuf/test/cases/YQL-19040.sql new file mode 100644 index 000000000000..895929c06c98 --- /dev/null +++ b/yql/essentials/udfs/common/protobuf/test/cases/YQL-19040.sql @@ -0,0 +1,62 @@ +/* +syntax='proto3'; + +import "yt/yt_proto/yt/formats/extension.proto"; + +message Test { + option (NYT.default_field_flags) = SERIALIZATION_YT; + + message Inner { + string a = 1; + } + map dict1 = 1 [(NYT.flags) = MAP_AS_DICT]; + map dict2 = 2 [(NYT.flags) = MAP_AS_OPTIONAL_DICT]; + map dict3 = 3 [(NYT.flags) = MAP_AS_LIST_OF_STRUCTS_LEGACY]; + map dict4 = 4 [(NYT.flags) = MAP_AS_LIST_OF_STRUCTS]; + map dict5 = 5 [(NYT.flags) = MAP_AS_DICT]; + map dict6 = 6 [(NYT.flags) = MAP_AS_OPTIONAL_DICT]; + map dict7 = 7 [(NYT.flags) = MAP_AS_LIST_OF_STRUCTS_LEGACY]; + map dict8 = 8 [(NYT.flags) = MAP_AS_LIST_OF_STRUCTS]; +} +*/ + +$config_json = @@{ + "name": "Test", + "format": "json", + "skip": 0, + "lists": { + "optional": false + }, + "meta": "eNrFWktvG9cVLp8iDyVqNJJtRo7rhHnYcWIqlV+q3KahyJFCl68OqSQyEAxGwyuKNjnDzAxty+iiQFddFVkVCIqi6CZFf0CBopsW3RcI0GZTBCjaAu1P6LLnPmY4fFm0m0cWDufc7zzuuefec869gj/dhhfaltXuko2+bbnW4eBoo0Ucw+70XcvOMZq8zBE5D5GtwMpup0uKPrBBXHkLokdIzIReiFxObb6cG2PKjXLUKVllHNl/RWF1yqgsQ9TUe1Ri6HJSZb/lDCz0deO+3iaZMCN7n/I3AVqkT8wWMY2TTAStSKoBivw6rPQHh92OoQVggLCYKvGB4hB8CZYfEv1+EJpi0DQlB4AFWOwRx0EDNPekTzJRNvsXJmY/PvOU4Goik5yHJDEHPS4hNsN/CiLGpSQomxCx4BD7QccgmTgTcGlCQIOPj8vw+HAqSfLIJabTsczMAhPyypRVJN3WuIghn3wTFqy+i7+cTALXJ7X5/NRAqHGM6oHlEkiONbANohlWi2gd88jKJJmAi5MTYcAC4koIU9POyLd8FuLOienqjzKLLELEV/Z3cVieJ8RuQ+yIzhID7Cl8wHlGnRh/RifmIWUSxyUtHhGROWMKONNkSEWfKaTeh2XfJM3WzbYXmxunWZJTPD6VsqlpMvItFwEsk1hHuL2MLsbJdC/VKGTCSxanGl3528NQW5gRKRW+ySaibR/SNqFxjy7mM0syI3KnzkwVbHxiS3bwU34JfILGwgrYKbToEatIW38M6VH3yGsQc1zddlkUxlT+IUsQwUOGnXIxlf6U3x5OOMIm/Orkio5IHp/3+i1YGpnAvKqzP4QzU0VjkKwNzI7pErtvExqxXFXm3wszYm4/iOZS1NXBJPFKMvGfBelH+F84+8c4rE3bM1O3L25/jOBDYjMnxVTxhTsi1tUPSRd3Q+hyevP1uXZlrkxZVM4pvwVRcURTCVfmk0D3ksr45POQpP/nsRFnNicogcaFvA4Jtk1axEtt/jcNrBY50gddV3ugdweEBTwGliC+S2nyRUjxXdVBnkfs9IypfKOVKIWqv+fgXhahyVRQAlN/a/zgvjB9ehN7CVMlQ1wTS693MysoIKGmObkmqNnfhiHKDpZlSDUP6opWrO3vlBUpJKcBGGG3XMs3pbD/Xao2b16XIj7DPidEg4Brm1IMA3aRCyi9rxQRER+lIGZBXoIko+zUamUp4ctsNNVSdU9K+jL31Np+XQJfQkVpNPJ7ipTyETsHTaUhLY6YhSqWfBVKdb8ipeUVWOIqPCOWx0hoqTQ0hEtZGSEgQs4WIMbCEMM9Xc7vKGWtVm+WatV8GX3n01TlB/slVSmi/wK0upJvIi2SNWBt2oE6dQsFYiE8IxaYrPFYyP4zDKtTkspUJd+DGI9lnmZfm5qdWGRPpFrGFyw1IjNKDSpiImA/mDj8eX68OU9+ZLSnSwKxKUngNqxMCJr7MP5xCDKznHPKkRgeORJvj3vwxdmLMLHWn4Tg7PSScqoNb0G8R9xjyyurXp2SrOnw+GILrmC2j8yqC7k1E5b+JAxnpgqfaugFgI7ZH7i8dOIncZJR2OFFT9mB649H2DhwEgNsDQ2NMkO/OWOmE4H5JkhGt0NMV3Ncm+i9jtlmqSaxHTvSuw5Rl/lwwxulHCyA7ABHfISDD/sc2Y+SkAoU4PKLsHhPf6BrXlPFPZGitLporN6ENQbBOaIio6s7DnNagkFlOlajQwVvRL4Bq4yjh7mp0+8SjbZ5Dks5vmUrFFERAGqRg2XhBcbWJiaxdZdo5MMBYjXdbGnHunOcWaMCdsKZkPocBe4JnMJgebP1DoLkbTjLpKBHcMKacUyM+9rAPdrKnA/qZxY2GKZAIfuIkBuwSBej13mMNls2y6HpKUdTwIO5mmCoYP+xHWvUFaWopjwpu5ZNA6pt+Q5O8YBqW5570VmGweeMvaloxpyMNOIsw9jjABHjDu6HM0NnBRlXJmY5zooa+yeTjPKIxv7JONstWOsf9yf5rgT5ZISMM77COnObGLhWrcy5IDwwIOcw/A2NmPohRoxu4w8nc5GBo649wC7CMBQ2mGdj8hVYsQ7vGTwiNRRz1HmUeZm5d5kOsHisM7L8Gsp2jnW7z45kBxeDZF7hUE6vemS6I5yHnSPXk3iJ7whGE9Iug0Q9MaL4MoOlkR7Ui8mAIodKX+OFGxKHGq/DWQrCg05v6a4eQL/B0NTtFTE4Yqc9ODzxA+sqt5PSvND60orz7DYsBuNeTgKPfCxIsAgq1Iq0fLmrYC2CZVS51FQ0db/aLFUUKRIo7O9EE69Kl7KfhiE92qnJ34Fz3rWKQ1ztYcdmG7Kn8+Tox8+aQDWI+x5idhlELsNF08IDAA8O3W5pwwstTTcwIB2LJ0JfyvOm1RDgYYbIC+hY+EZmhS9W1z29j/Hr2iesPk+oCSQo9PsraZPQmwkpif8mJcj+IwKLwXqdtj8Gy1ghdqa99MTqPlegqWw7zotjlXPSMoIGG+HFSEIVX/IexO85THacyX75ybLvNJjw5J2GVq2plXxZFezycxDt6o9PRpMeI827CCiBXtCNphpG+hI3wwbEmL9kAOEx6RtyAqKFmko3BO4ATtXqJaWAeyJ7A+LcCXSz+G5AJv4pZIS80f3KjqJK4dGljkqxrIO7MFCHfzXN+B9CkArU1bQg0rtd66Gmdzu6I0IDGClPKfMu3Ve0RWJSPPuLEEjjhe2YmaGv08zsz0OQHq1mx8x78Ws17+9hWBqpYee17kNY6bRIr2+59PJc65IHpJvJskNj48lVcq405CtTtu3VUlGp1GtNpVo40Par36/W3quqUmcM9iVu+zpI40bJ52CaWbizV2G5WsOciIlR2d1VCs0Gv/fw0c2RDZ79OAKrUyzBY5x3LLyJujqP9TlaM9SxlRQNDtZC6CXT7Rx1sJ7nPThvY5aHdH6l9AbIfcvpuJ0H9Ereu3yibU1UlbyRkun6aJO09TE0PcwjquSN+GisX1rWgNZ6HEdzR0hNcZoPEVX88NZrEUsxRuOQS7Cst9s2Fe4J4n1J2icz4PodSHh+oKmaegJLJ9Zsh+lFmOkNotKOow0v8cM4nlBTHce/AM1+ggXL6CME9i6JroVBTjn4C9jlU94tcmWBV33O9T+HIOGRMd1G+7p7zMTFdsJSSGXflI4VoMlCQNDpN13XLtFbrOmxej1cScdbV0EvCDJ9C3NtvdMdwUYZVvIGfPA2POfJbWEVig1Va8gUZ5cb5wSgKMY93uynIVjx2rSW76wKgG6alht012QoT/Dl8j6TGhCw3gMYjsx0G+Yp8cLEnil5Yw+cRPs5ev1ySNodU9wb8w/v+iXqX7/s/DSEHZvVG7d3Rxq7XXDeCd19q91xjweHOcRvtK2ubraH76zsh3EV+6mrbSvw6np7+PO/odCvwpG9+s5vwut7XF3dc49KjrrEoFOGj5fgKpaaNmkNDLJx4m6wM+AIuwSuxNnww1k86UaqB831U19+aaCvKO/Zer9PbFbD7Xb1dvajMERp+pQXIJKvHuDptgJLteY7iqoVauX9SpWebYuQoNeg9IoWu4BlrBnolyhtIuhsqaGopXy5dDdP7zK1g6YUldfh7Ci1rtaatZ39XSlGdXhXnFq51GhKcaZW3IRy0gLu3guVfF3LNxhBq+1Slft43GplZS9fOJASVMl0iJSkdoqxYqnQlEDOwJog+JrYSIpNEAuzYhG7nsXsfVj1HCVaGeaqpvDU63CpqNRVpUBvY7VGTW1quyWlXGxQySUxTySVFfTmRTgfROwc8F9eIRjKfne4Kqz+Y6ouC1WYaRpKPa+iJiFBWpRTsPBuHh2Li7G07UKGbgHNe1LgrwZHKMSRn/hcm/nL7+lbZHrzfA7jJzdhA78cPXPE3vKZbH/ImdDK3kvn0vrXqVr9eJzU6g85249hfUSr11XOo/czoffCiN7A8nLN5wKaA4POtgoxrubJTykzvDo2Py5q+21IGVZ30OOvOKdJpp5jxxznoQlwew+W75MT7SmkfCakLCFfYSjIhNVpEXTaK+x8QbTSmgiggL5g7Jyqb67w8fQFQofA0mi0nKppzoDx/mKEq/nAe7abFSvBFm8+5/FHPy6+AvID3e7opue1GSs+osWLG0mwMq/QRd95825Oz53o9D3xqkv0Xs4ebDwx9cDfFiBx4nJF60+XprI/W4BokzgulpaxVsdwvyWKBSlHqbkiJbELlp3oLz8/DyoHeehNUSUH0JtDdIqjNz30NfF3FgH0tSE6wdHXPPR18WgUQF8fopMcfd1D3xB/ORFA35iw+4aHvin+cCSAvjlh900PfUv8rU4AfWvC7lseekv8uUUAvTVh99b6GYiVTCy/MMuFdPEoENLXFYChz2lphCeCGKU/Mfv6b3r05SPFlTBJ4tVuO7wV8sRsfjFirn0xYq7/n2K2uJgbs8SsBcUkp3DefGbOW8/MufXUnNvRX39+PnoY54/7/wMXV6Lr", + "view": { + "recursion": "bytes", + "enum": "number", + "yt_mode": true, + "deterministic": true + } +} +@@; + +$config_bin = @@{ + "name": "Test", + "skip": 0, + "lists": { + "optional": false + }, + "meta": "eNrFWktvG9cVLp8iDyVqNJJtRo7rhHnYcWIqlV+q3KahyJFCl68OqSQyEAxGwyuKNjnDzAxty+iiQFddFVkVCIqi6CZFf0CBopsW3RcI0GZTBCjaAu1P6LLnPmY4fFm0m0cWDufc7zzuuefec869gj/dhhfaltXuko2+bbnW4eBoo0Ucw+70XcvOMZq8zBE5D5GtwMpup0uKPrBBXHkLokdIzIReiFxObb6cG2PKjXLUKVllHNl/RWF1yqgsQ9TUe1Ri6HJSZb/lDCz0deO+3iaZMCN7n/I3AVqkT8wWMY2TTAStSKoBivw6rPQHh92OoQVggLCYKvGB4hB8CZYfEv1+EJpi0DQlB4AFWOwRx0EDNPekTzJRNvsXJmY/PvOU4Goik5yHJDEHPS4hNsN/CiLGpSQomxCx4BD7QccgmTgTcGlCQIOPj8vw+HAqSfLIJabTsczMAhPyypRVJN3WuIghn3wTFqy+i7+cTALXJ7X5/NRAqHGM6oHlEkiONbANohlWi2gd88jKJJmAi5MTYcAC4koIU9POyLd8FuLOienqjzKLLELEV/Z3cVieJ8RuQ+yIzhID7Cl8wHlGnRh/RifmIWUSxyUtHhGROWMKONNkSEWfKaTeh2XfJM3WzbYXmxunWZJTPD6VsqlpMvItFwEsk1hHuL2MLsbJdC/VKGTCSxanGl3528NQW5gRKRW+ySaibR/SNqFxjy7mM0syI3KnzkwVbHxiS3bwU34JfILGwgrYKbToEatIW38M6VH3yGsQc1zddlkUxlT+IUsQwUOGnXIxlf6U3x5OOMIm/Orkio5IHp/3+i1YGpnAvKqzP4QzU0VjkKwNzI7pErtvExqxXFXm3wszYm4/iOZS1NXBJPFKMvGfBelH+F84+8c4rE3bM1O3L25/jOBDYjMnxVTxhTsi1tUPSRd3Q+hyevP1uXZlrkxZVM4pvwVRcURTCVfmk0D3ksr45POQpP/nsRFnNicogcaFvA4Jtk1axEtt/jcNrBY50gddV3ugdweEBTwGliC+S2nyRUjxXdVBnkfs9IypfKOVKIWqv+fgXhahyVRQAlN/a/zgvjB9ehN7CVMlQ1wTS693MysoIKGmObkmqNnfhiHKDpZlSDUP6opWrO3vlBUpJKcBGGG3XMs3pbD/Xao2b16XIj7DPidEg4Brm1IMA3aRCyi9rxQRER+lIGZBXoIko+zUamUp4ctsNNVSdU9K+jL31Np+XQJfQkVpNPJ7ipTyETsHTaUhLY6YhSqWfBVKdb8ipeUVWOIqPCOWx0hoqTQ0hEtZGSEgQs4WIMbCEMM9Xc7vKGWtVm+WatV8GX3n01TlB/slVSmi/wK0upJvIi2SNWBt2oE6dQsFYiE8IxaYrPFYyP4zDKtTkspUJd+DGI9lnmZfm5qdWGRPpFrGFyw1IjNKDSpiImA/mDj8eX68OU9+ZLSnSwKxKUngNqxMCJr7MP5xCDKznHPKkRgeORJvj3vwxdmLMLHWn4Tg7PSScqoNb0G8R9xjyyurXp2SrOnw+GILrmC2j8yqC7k1E5b+JAxnpgqfaugFgI7ZH7i8dOIncZJR2OFFT9mB649H2DhwEgNsDQ2NMkO/OWOmE4H5JkhGt0NMV3Ncm+i9jtlmqSaxHTvSuw5Rl/lwwxulHCyA7ABHfISDD/sc2Y+SkAoU4PKLsHhPf6BrXlPFPZGitLporN6ENQbBOaIio6s7DnNagkFlOlajQwVvRL4Bq4yjh7mp0+8SjbZ5Dks5vmUrFFERAGqRg2XhBcbWJiaxdZdo5MMBYjXdbGnHunOcWaMCdsKZkPocBe4JnMJgebP1DoLkbTjLpKBHcMKacUyM+9rAPdrKnA/qZxY2GKZAIfuIkBuwSBej13mMNls2y6HpKUdTwIO5mmCoYP+xHWvUFaWopjwpu5ZNA6pt+Q5O8YBqW5570VmGweeMvaloxpyMNOIsw9jjABHjDu6HM0NnBRlXJmY5zooa+yeTjPKIxv7JONstWOsf9yf5rgT5ZISMM77COnObGLhWrcy5IDwwIOcw/A2NmPohRoxu4w8nc5GBo649wC7CMBQ2mGdj8hVYsQ7vGTwiNRRz1HmUeZm5d5kOsHisM7L8Gsp2jnW7z45kBxeDZF7hUE6vemS6I5yHnSPXk3iJ7whGE9Iug0Q9MaL4MoOlkR7Ui8mAIodKX+OFGxKHGq/DWQrCg05v6a4eQL/B0NTtFTE4Yqc9ODzxA+sqt5PSvND60orz7DYsBuNeTgKPfCxIsAgq1Iq0fLmrYC2CZVS51FQ0db/aLFUUKRIo7O9EE69Kl7KfhiE92qnJ34Fz3rWKQ1ztYcdmG7Kn8+Tox8+aQDWI+x5idhlELsNF08IDAA8O3W5pwwstTTcwIB2LJ0JfyvOm1RDgYYbIC+hY+EZmhS9W1z29j/Hr2iesPk+oCSQo9PsraZPQmwkpif8mJcj+IwKLwXqdtj8Gy1ghdqa99MTqPlegqWw7zotjlXPSMoIGG+HFSEIVX/IexO85THacyX75ybLvNJjw5J2GVq2plXxZFezycxDt6o9PRpMeI827CCiBXtCNphpG+hI3wwbEmL9kAOEx6RtyAqKFmko3BO4ATtXqJaWAeyJ7A+LcCXSz+G5AJv4pZIS80f3KjqJK4dGljkqxrIO7MFCHfzXN+B9CkArU1bQg0rtd66Gmdzu6I0IDGClPKfMu3Ve0RWJSPPuLEEjjhe2YmaGv08zsz0OQHq1mx8x78Ws17+9hWBqpYee17kNY6bRIr2+59PJc65IHpJvJskNj48lVcq405CtTtu3VUlGp1GtNpVo40Par36/W3quqUmcM9iVu+zpI40bJ52CaWbizV2G5WsOciIlR2d1VCs0Gv/fw0c2RDZ79OAKrUyzBY5x3LLyJujqP9TlaM9SxlRQNDtZC6CXT7Rx1sJ7nPThvY5aHdH6l9AbIfcvpuJ0H9Ereu3yibU1UlbyRkun6aJO09TE0PcwjquSN+GisX1rWgNZ6HEdzR0hNcZoPEVX88NZrEUsxRuOQS7Cst9s2Fe4J4n1J2icz4PodSHh+oKmaegJLJ9Zsh+lFmOkNotKOow0v8cM4nlBTHce/AM1+ggXL6CME9i6JroVBTjn4C9jlU94tcmWBV33O9T+HIOGRMd1G+7p7zMTFdsJSSGXflI4VoMlCQNDpN13XLtFbrOmxej1cScdbV0EvCDJ9C3NtvdMdwUYZVvIGfPA2POfJbWEVig1Va8gUZ5cb5wSgKMY93uynIVjx2rSW76wKgG6alht012QoT/Dl8j6TGhCw3gMYjsx0G+Yp8cLEnil5Yw+cRPs5ev1ySNodU9wb8w/v+iXqX7/s/DSEHZvVG7d3Rxq7XXDeCd19q91xjweHOcRvtK2ubraH76zsh3EV+6mrbSvw6np7+PO/odCvwpG9+s5vwut7XF3dc49KjrrEoFOGj5fgKpaaNmkNDLJx4m6wM+AIuwSuxNnww1k86UaqB831U19+aaCvKO/Zer9PbFbD7Xb1dvajMERp+pQXIJKvHuDptgJLteY7iqoVauX9SpWebYuQoNeg9IoWu4BlrBnolyhtIuhsqaGopXy5dDdP7zK1g6YUldfh7Ci1rtaatZ39XSlGdXhXnFq51GhKcaZW3IRy0gLu3guVfF3LNxhBq+1Slft43GplZS9fOJASVMl0iJSkdoqxYqnQlEDOwJog+JrYSIpNEAuzYhG7nsXsfVj1HCVaGeaqpvDU63CpqNRVpUBvY7VGTW1quyWlXGxQySUxTySVFfTmRTgfROwc8F9eIRjKfne4Kqz+Y6ouC1WYaRpKPa+iJiFBWpRTsPBuHh2Li7G07UKGbgHNe1LgrwZHKMSRn/hcm/nL7+lbZHrzfA7jJzdhA78cPXPE3vKZbH/ImdDK3kvn0vrXqVr9eJzU6g85249hfUSr11XOo/czoffCiN7A8nLN5wKaA4POtgoxrubJTykzvDo2Py5q+21IGVZ30OOvOKdJpp5jxxznoQlwew+W75MT7SmkfCakLCFfYSjIhNVpEXTaK+x8QbTSmgiggL5g7Jyqb67w8fQFQofA0mi0nKppzoDx/mKEq/nAe7abFSvBFm8+5/FHPy6+AvID3e7opue1GSs+osWLG0mwMq/QRd95825Oz53o9D3xqkv0Xs4ebDwx9cDfFiBx4nJF60+XprI/W4BokzgulpaxVsdwvyWKBSlHqbkiJbELlp3oLz8/DyoHeehNUSUH0JtDdIqjNz30NfF3FgH0tSE6wdHXPPR18WgUQF8fopMcfd1D3xB/ORFA35iw+4aHvin+cCSAvjlh900PfUv8rU4AfWvC7lseekv8uUUAvTVh99b6GYiVTCy/MMuFdPEoENLXFYChz2lphCeCGKU/Mfv6b3r05SPFlTBJ4tVuO7wV8sRsfjFirn0xYq7/n2K2uJgbs8SsBcUkp3DefGbOW8/MufXUnNvRX39+PnoY54/7/wMXV6Lr", + "view": { + "recursion": "bytes", + "enum": "number", + "yt_mode": true, + "deterministic": true + } +} +@@; + +$dat = "{\"dict1\":[{\"key\":\"k1\",\"value\":{\"a\":\"1\"}},{\"key\":\"k2\",\"value\":{\"a\":\"0\"}}],\"dict2\":[{\"key\":\"k2\",\"value\":{\"a\":\"2\"}}],\"dict3\":[{\"key\":\"k3\",\"value\":{\"a\":\"3\"}}],\"dict4\":[{\"key\":\"k4\",\"value\":{\"a\":\"4\"}}],\"dict5\":[{\"key\":\"k5\",\"value\":\"v5\"}],\"dict6\":[{\"key\":\"k6\",\"value\":\"v6\"}],\"dict7\":[{\"key\":\"k7\",\"value\":\"v7\"}],\"dict8\":[{\"key\":\"k8\",\"value\":\"v8\"}]}"; + +$udfParseJson = Udf(Protobuf::Parse, $config_json as TypeConfig); +$udfParseBin = Udf(Protobuf::Parse, $config_bin as TypeConfig); +$udfSerializeBin = Udf(Protobuf::Serialize, $config_bin as TypeConfig); + +SELECT Ensure("Success", $udfParseJson($dat) == $udfParseBin($udfSerializeBin($udfParseJson($dat))), "Fail"); \ No newline at end of file From fab0037c69a3d2c1d81904e6174dcb4317ba02a2 Mon Sep 17 00:00:00 2001 From: bulatman Date: Fri, 10 Jan 2025 01:27:33 +0300 Subject: [PATCH 5/5] YT: Removed extra trailing comma in std::tuple formatting commit_hash:19b37bd3d7e3b4d80e4844676c881bbef1fec76b --- library/cpp/yt/string/format-inl.h | 2 +- library/cpp/yt/string/unittests/format_ut.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/library/cpp/yt/string/format-inl.h b/library/cpp/yt/string/format-inl.h index e90d68bfe12a..b18c435cb862 100644 --- a/library/cpp/yt/string/format-inl.h +++ b/library/cpp/yt/string/format-inl.h @@ -722,7 +722,7 @@ void FormatValue(TStringBuilderBase* builder, const std::tuple& value, TS [&] (std::index_sequence) { ([&] { FormatValue(builder, std::get(value), spec); - if constexpr (Idx != sizeof...(Ts)) { + if constexpr (Idx != sizeof...(Ts) - 1) { builder->AppendString(TStringBuf(", ")); } } (), ...); diff --git a/library/cpp/yt/string/unittests/format_ut.cpp b/library/cpp/yt/string/unittests/format_ut.cpp index 8aca8c8e2901..e6b5eb7cfec8 100644 --- a/library/cpp/yt/string/unittests/format_ut.cpp +++ b/library/cpp/yt/string/unittests/format_ut.cpp @@ -239,6 +239,13 @@ TEST(TFormatTest, Pointers) } } +TEST(TFormatTest, Tuples) +{ + EXPECT_EQ("{}", Format("%v", std::tuple())); + EXPECT_EQ("{1, 2, 3}", Format("%v", std::tuple(1, 2, 3))); + EXPECT_EQ("{1, 2}", Format("%v", std::pair(1, 2))); +} + TEST(TFormatTest, LazyMultiValueFormatter) { int i = 1;