From 145dd60ea17c5d3fb35c665a555eaece356c6a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20V=C3=A1rady?= Date: Mon, 12 Jun 2023 17:05:39 +0200 Subject: [PATCH] grpc: move protos to a separate library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - when compiling protos into a shared library, the shared library can not be reloaded: https://github.com/protocolbuffers/protobuf/issues/4126 - it is not recommended to have multiple shared libraries with protos compiled in them: https://github.com/protocolbuffers/protobuf/issues/1489 Signed-off-by: László Várady --- .gitmodules | 2 +- modules/grpc/CMakeLists.txt | 1 + modules/grpc/Makefile.am | 2 + modules/grpc/otel/CMakeLists.txt | 33 +-------- modules/grpc/otel/Makefile.am | 59 ++-------------- modules/grpc/otel/tests/Makefile.am | 16 +++-- modules/grpc/protos/CMakeLists.txt | 34 +++++++++ modules/grpc/protos/Makefile.am | 69 +++++++++++++++++++ .../grpc/{otel => protos}/opentelemetry-proto | 0 packaging/debian/syslog-ng-mod-grpc.install | 1 + 10 files changed, 125 insertions(+), 92 deletions(-) create mode 100644 modules/grpc/protos/CMakeLists.txt create mode 100644 modules/grpc/protos/Makefile.am rename modules/grpc/{otel => protos}/opentelemetry-proto (100%) diff --git a/.gitmodules b/.gitmodules index e59bf4a4ebb..2fb422a68ba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,5 +3,5 @@ url = https://github.com/bazsi/ivykis.git ignore = dirty [submodule "modules/grpc/otel/opentelemetry-proto"] - path = modules/grpc/otel/opentelemetry-proto + path = modules/grpc/protos/opentelemetry-proto url = https://github.com/open-telemetry/opentelemetry-proto.git diff --git a/modules/grpc/CMakeLists.txt b/modules/grpc/CMakeLists.txt index 02a1023ca64..5685e9db75d 100644 --- a/modules/grpc/CMakeLists.txt +++ b/modules/grpc/CMakeLists.txt @@ -27,4 +27,5 @@ set(MODULE_GRPC_LIBS gRPC::grpc++ protobuf::libprotobuf) +add_subdirectory(protos) add_subdirectory(otel) diff --git a/modules/grpc/Makefile.am b/modules/grpc/Makefile.am index f669825d151..0c45b413dd2 100644 --- a/modules/grpc/Makefile.am +++ b/modules/grpc/Makefile.am @@ -1,3 +1,5 @@ +include modules/grpc/protos/Makefile.am + include modules/grpc/otel/Makefile.am if ENABLE_GRPC diff --git a/modules/grpc/otel/CMakeLists.txt b/modules/grpc/otel/CMakeLists.txt index 283f24bd684..7d0c398d1e4 100644 --- a/modules/grpc/otel/CMakeLists.txt +++ b/modules/grpc/otel/CMakeLists.txt @@ -1,30 +1,3 @@ -set(OTEL_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/opentelemetry-proto") -set(OTEL_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/proto") - -set(OTEL_PROTO_SOURCES - opentelemetry/proto/common/v1/common.proto - opentelemetry/proto/logs/v1/logs.proto - opentelemetry/proto/metrics/v1/metrics.proto - opentelemetry/proto/trace/v1/trace.proto - opentelemetry/proto/resource/v1/resource.proto) - -set(OTEL_PROTO_GRPC_SOURCES - opentelemetry/proto/collector/logs/v1/logs_service.proto - opentelemetry/proto/collector/metrics/v1/metrics_service.proto - opentelemetry/proto/collector/trace/v1/trace_service.proto) - -protobuf_generate_cpp( - PROTO_PATH ${OTEL_PROTO_SRCDIR} - CPP_OUT ${OTEL_PROTO_BUILDDIR} - OUT_SRCS OTEL_PROTO_GENERATED_SOURCES - PROTOS ${OTEL_PROTO_SOURCES}) - -protobuf_generate_cpp_grpc( - PROTO_PATH ${OTEL_PROTO_SRCDIR} - CPP_OUT ${OTEL_PROTO_BUILDDIR} - OUT_SRCS OTEL_PROTO_GENERATED_GRPC_SOURCES - PROTOS ${OTEL_PROTO_GRPC_SOURCES}) - set(OTEL_CPP_SOURCES otel-source.cpp otel-source.hpp @@ -43,9 +16,7 @@ set(OTEL_CPP_SOURCES otel-dest.cpp otel-dest.h otel-dest-worker.hpp - otel-dest-worker.cpp - ${OTEL_PROTO_GENERATED_SOURCES} - ${OTEL_PROTO_GENERATED_GRPC_SOURCES}) + otel-dest-worker.cpp) set(OTEL_SOURCES otel-plugin.c @@ -56,7 +27,7 @@ set(OTEL_SOURCES add_module( TARGET otel-cpp SOURCES ${OTEL_CPP_SOURCES} - DEPENDS ${MODULE_GRPC_LIBS} + DEPENDS ${MODULE_GRPC_LIBS} grpc-protos INCLUDES ${OTEL_PROTO_BUILDDIR} LIBRARY_TYPE STATIC ) diff --git a/modules/grpc/otel/Makefile.am b/modules/grpc/otel/Makefile.am index ac1d5d1c068..6cad2c8613f 100644 --- a/modules/grpc/otel/Makefile.am +++ b/modules/grpc/otel/Makefile.am @@ -1,46 +1,7 @@ -OTEL_PROTO_SRCDIR = $(top_srcdir)/modules/grpc/otel/opentelemetry-proto -OTEL_PROTO_BUILDDIR = $(top_builddir)/modules/grpc/otel/proto - if ENABLE_GRPC -OTEL_PROTO_PROTOC_FLAGS = $(PROTOC_GEN_GRPC_CPP_PLUGIN_FLAGS) --proto_path=$(OTEL_PROTO_SRCDIR) --cpp_out=$(OTEL_PROTO_BUILDDIR) - -modules/grpc/otel/proto/%.grpc.pb.cc modules/grpc/otel/proto/%.grpc.pb.h: - mkdir -p $(OTEL_PROTO_BUILDDIR) - $(PROTOC) $(OTEL_PROTO_PROTOC_FLAGS) --grpc-cpp_out=$(OTEL_PROTO_BUILDDIR) $*.proto - -modules/grpc/otel/proto/%.pb.cc modules/grpc/otel/proto/%.pb.h: - mkdir -p $(OTEL_PROTO_BUILDDIR) - $(PROTOC) $(OTEL_PROTO_PROTOC_FLAGS) $*.proto - -OTEL_PROTO_BUILT_SOURCES = \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/common/v1/common.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/common/v1/common.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/resource/v1/resource.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/resource/v1/resource.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/logs/v1/logs.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/logs/v1/logs.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/metrics/v1/metrics.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/metrics/v1/metrics.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/trace/v1/trace.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/trace/v1/trace.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.cc \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.pb.h \ - $(OTEL_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc - noinst_LTLIBRARIES += modules/grpc/otel/libotel_cpp.la -nodist_modules_grpc_otel_libotel_cpp_la_SOURCES = $(OTEL_PROTO_BUILT_SOURCES) - modules_grpc_otel_libotel_cpp_la_SOURCES = \ modules/grpc/otel/otel-source.h \ modules/grpc/otel/otel-source.hpp \ @@ -65,14 +26,10 @@ modules_grpc_otel_libotel_cpp_la_CXXFLAGS = \ $(AM_CXXFLAGS) \ $(PROTOBUF_CFLAGS) \ $(GRPCPP_CFLAGS) \ - -I$(OTEL_PROTO_BUILDDIR) \ + -I$(OPENTELEMETRY_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc/otel \ -I$(top_builddir)/modules/grpc/otel -if ENABLE_EXTRA_WARNINGS -modules_grpc_otel_libotel_cpp_la_CXXFLAGS += -Wno-switch-default -endif - modules_grpc_otel_libotel_cpp_la_LIBADD = $(MODULE_DEPS_LIBS) $(PROTOBUF_LIBS) $(GRPCPP_LIBS) modules_grpc_otel_libotel_cpp_la_LDFLAGS = $(MODULE_LDFLAGS) modules_grpc_otel_libotel_cpp_la_DEPENDENCIES = $(MODULE_DEPS_LIBS) @@ -92,6 +49,7 @@ modules_grpc_otel_libotel_la_CPPFLAGS = \ modules_grpc_otel_libotel_la_LIBADD = \ $(MODULE_DEPS_LIBS) \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ -lstdc++ @@ -99,6 +57,7 @@ modules_grpc_otel_libotel_la_LDFLAGS = $(MODULE_LDFLAGS) modules_grpc_otel_libotel_la_DEPENDENCIES = \ $(MODULE_DEPS_LIBS) \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la \ $(top_builddir)/modules/grpc/otel/libotel_cpp.la modules/grpc/otel modules/grpc/otel/ mod-otel: modules/grpc/otel/libotel.la @@ -106,22 +65,14 @@ else modules/grpc/otel modules/grpc/otel/ mod-otel: endif -CLEANFILES += \ - $(OTEL_PROTO_BUILT_SOURCES) - BUILT_SOURCES += \ modules/grpc/otel/otel-grammar.y \ modules/grpc/otel/otel-grammar.c \ - modules/grpc/otel/otel-grammar.h \ - $(OTEL_PROTO_BUILT_SOURCES) - -NODIST_BUILT_SOURCES += \ - $(OTEL_PROTO_BUILT_SOURCES) + modules/grpc/otel/otel-grammar.h EXTRA_DIST += \ modules/grpc/otel/otel-grammar.ym \ - modules/grpc/otel/CMakeLists.txt \ - $(OTEL_PROTO_SRCDIR) + modules/grpc/otel/CMakeLists.txt .PHONY: modules/grpc/otel/ mod-otel diff --git a/modules/grpc/otel/tests/Makefile.am b/modules/grpc/otel/tests/Makefile.am index 7ebc2489d63..db380c21a02 100644 --- a/modules/grpc/otel/tests/Makefile.am +++ b/modules/grpc/otel/tests/Makefile.am @@ -10,33 +10,37 @@ modules_grpc_otel_tests_test_otel_protobuf_parser_SOURCES = \ modules/grpc/otel/tests/test-otel-protobuf-parser.cpp modules_grpc_otel_tests_test_otel_protobuf_parser_DEPENDENCIES = \ - $(top_builddir)/modules/grpc/otel/libotel_cpp.la + $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la modules_grpc_otel_tests_test_otel_protobuf_parser_CXXFLAGS = \ $(TEST_CXXFLAGS) \ - -I$(OTEL_PROTO_BUILDDIR) \ + -I$(OPENTELEMETRY_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc/otel \ -I$(top_builddir)/modules/grpc/otel modules_grpc_otel_tests_test_otel_protobuf_parser_LDADD = \ $(TEST_LDADD) \ - $(top_builddir)/modules/grpc/otel/libotel_cpp.la + $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la modules_grpc_otel_tests_test_otel_protobuf_formatter_SOURCES = \ modules/grpc/otel/tests/test-otel-protobuf-formatter.cpp modules_grpc_otel_tests_test_otel_protobuf_formatter_DEPENDENCIES = \ - $(top_builddir)/modules/grpc/otel/libotel_cpp.la + $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la modules_grpc_otel_tests_test_otel_protobuf_formatter_CXXFLAGS = \ $(TEST_CXXFLAGS) \ - -I$(OTEL_PROTO_BUILDDIR) \ + -I$(OPENTELEMETRY_PROTO_BUILDDIR) \ -I$(top_srcdir)/modules/grpc/otel \ -I$(top_builddir)/modules/grpc/otel modules_grpc_otel_tests_test_otel_protobuf_formatter_LDADD = \ $(TEST_LDADD) \ - $(top_builddir)/modules/grpc/otel/libotel_cpp.la + $(top_builddir)/modules/grpc/otel/libotel_cpp.la \ + $(top_builddir)/modules/grpc/protos/libgrpc-protos.la endif diff --git a/modules/grpc/protos/CMakeLists.txt b/modules/grpc/protos/CMakeLists.txt new file mode 100644 index 00000000000..19492686488 --- /dev/null +++ b/modules/grpc/protos/CMakeLists.txt @@ -0,0 +1,34 @@ +set(OTEL_PROTO_SRCDIR "${CMAKE_CURRENT_SOURCE_DIR}/opentelemetry-proto") +set(OTEL_PROTO_BUILDDIR "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry-proto") +set(OTEL_PROTO_BUILDDIR ${OTEL_PROTO_BUILDDIR} PARENT_SCOPE) + +set(GRPC_PROTO_SOURCES + opentelemetry/proto/common/v1/common.proto + opentelemetry/proto/logs/v1/logs.proto + opentelemetry/proto/metrics/v1/metrics.proto + opentelemetry/proto/trace/v1/trace.proto + opentelemetry/proto/resource/v1/resource.proto) + +set(GRPC_PROTO_GRPC_SOURCES + opentelemetry/proto/collector/logs/v1/logs_service.proto + opentelemetry/proto/collector/metrics/v1/metrics_service.proto + opentelemetry/proto/collector/trace/v1/trace_service.proto) + +protobuf_generate_cpp( + PROTO_PATH ${OTEL_PROTO_SRCDIR} + CPP_OUT ${OTEL_PROTO_BUILDDIR} + OUT_SRCS GRPC_PROTO_GENERATED_SOURCES + PROTOS ${GRPC_PROTO_SOURCES}) + +protobuf_generate_cpp_grpc( + PROTO_PATH ${OTEL_PROTO_SRCDIR} + CPP_OUT ${OTEL_PROTO_BUILDDIR} + OUT_SRCS GRPC_PROTO_GENERATED_GRPC_SOURCES + PROTOS ${GRPC_PROTO_GRPC_SOURCES}) + +add_module( + TARGET grpc-protos + SOURCES ${GRPC_PROTO_GENERATED_SOURCES} ${GRPC_PROTO_GENERATED_GRPC_SOURCES} + DEPENDS ${MODULE_GRPC_LIBS} + INCLUDES ${OTEL_PROTO_BUILDDIR} +) diff --git a/modules/grpc/protos/Makefile.am b/modules/grpc/protos/Makefile.am new file mode 100644 index 00000000000..23267027ff0 --- /dev/null +++ b/modules/grpc/protos/Makefile.am @@ -0,0 +1,69 @@ +GRPC_PROTOS_SRCDIR = $(top_srcdir)/modules/grpc/protos +GRPC_PROTOS_BUILDDIR = $(top_builddir)/modules/grpc/protos + +if ENABLE_GRPC + +OPENTELEMETRY_PROTO_SRCDIR = $(GRPC_PROTOS_SRCDIR)/opentelemetry-proto +OPENTELEMETRY_PROTO_BUILDDIR = $(GRPC_PROTOS_BUILDDIR)/otlp + +OPENTELEMETRY_PROTO_PROTOC_FLAGS = $(PROTOC_GEN_GRPC_CPP_PLUGIN_FLAGS) --proto_path=$(OPENTELEMETRY_PROTO_SRCDIR) --cpp_out=$(OPENTELEMETRY_PROTO_BUILDDIR) + +modules/grpc/protos/otlp/%.grpc.pb.cc modules/grpc/protos/otlp/%.grpc.pb.h: + mkdir -p $(OPENTELEMETRY_PROTO_BUILDDIR) + $(PROTOC) $(OPENTELEMETRY_PROTO_PROTOC_FLAGS) --grpc-cpp_out=$(OPENTELEMETRY_PROTO_BUILDDIR) $*.proto + +modules/grpc/protos/otlp/%.pb.cc modules/grpc/protos/otlp/%.pb.h: + mkdir -p $(OPENTELEMETRY_PROTO_BUILDDIR) + $(PROTOC) $(OPENTELEMETRY_PROTO_PROTOC_FLAGS) $*.proto + +GRPC_PROTOS_BUILT_SOURCES = \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/common/v1/common.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/common/v1/common.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/resource/v1/resource.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/resource/v1/resource.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/logs/v1/logs.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/logs/v1/logs.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/metrics/v1/metrics.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/metrics/v1/metrics.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/trace/v1/trace.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/trace/v1/trace.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.grpc.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/logs/v1/logs_service.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.grpc.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/metrics/v1/metrics_service.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.grpc.pb.cc \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.pb.h \ + $(OPENTELEMETRY_PROTO_BUILDDIR)/opentelemetry/proto/collector/trace/v1/trace_service.pb.cc + +lib_LTLIBRARIES += modules/grpc/protos/libgrpc-protos.la + +nodist_modules_grpc_protos_libgrpc_protos_la_SOURCES = $(GRPC_PROTOS_BUILT_SOURCES) + +modules_grpc_protos_libgrpc_protos_la_CXXFLAGS = \ + $(AM_CXXFLAGS) \ + $(PROTOBUF_CLFAGS) \ + $(GRPCPP_CFLAGS) \ + -I$(OPENTELEMETRY_PROTO_SRCDIR) \ + -I$(OPENTELEMETRY_PROTO_BUILDDIR) + +modules_grpc_protos_libgrpc_protos_la_LIBADD = $(MODULE_DEPS_LIBS) $(PROTOBUF_LIBS) $(GRPCPP_LIBS) -lstdc++ +modules_grpc_protos_libgrpc_protos_la_DEPENDENCIES = $(MODULE_DEPS_LIBS) + +endif + +CLEANFILES += \ + $(GRPC_PROTOS_BUILT_SOURCES) + +BUILT_SOURCES += \ + $(GRPC_PROTOS_BUILT_SOURCES) + +NODIST_BUILT_SOURCES += \ + $(GRPC_PROTOS_BUILT_SOURCES) + +EXTRA_DIST += \ + $(OPENTELEMETRY_PROTO_SRCDIR) diff --git a/modules/grpc/otel/opentelemetry-proto b/modules/grpc/protos/opentelemetry-proto similarity index 100% rename from modules/grpc/otel/opentelemetry-proto rename to modules/grpc/protos/opentelemetry-proto diff --git a/packaging/debian/syslog-ng-mod-grpc.install b/packaging/debian/syslog-ng-mod-grpc.install index 787955e114c..1dd33741027 100644 --- a/packaging/debian/syslog-ng-mod-grpc.install +++ b/packaging/debian/syslog-ng-mod-grpc.install @@ -1 +1,2 @@ +usr/lib/syslog-ng/*/libgrpc-protos.so usr/lib/syslog-ng/*/libotel.so