contrib: add qatzstd compressor

api/BUILD

@@ -73,6 +73,7 @@ proto_library(
     visibility = ["//visibility:public"],
     deps = [
         "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
+        "//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/checksum/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",
         "//contrib/envoy/extensions/filters/http/golang/v3alpha:pkg",

api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_xds//udpa/annotations:pkg"],
+)

api/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha/qatzstd.proto

@@ -0,0 +1,64 @@
+syntax = "proto3";
+
+package envoy.extensions.compression.qatzstd.compressor.v3alpha;
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.compression.qatzstd.compressor.v3alpha";
+option java_outer_classname = "QatzstdProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Qatzstd Compressor]
+// Qatzstd :ref:`configuration overview <config_qatzstd>`.
+// [#extension: envoy.compression.qatzstd.compressor]
+
+// [#next-free-field: 8]
+message Qatzstd {
+  // Reference to http://facebook.github.io/zstd/zstd_manual.html
+  enum Strategy {
+    DEFAULT = 0;
+    FAST = 1;
+    DFAST = 2;
+    GREEDY = 3;
+    LAZY = 4;
+    LAZY2 = 5;
+    BTLAZY2 = 6;
+    BTOPT = 7;
+    BTULTRA = 8;
+    BTULTRA2 = 9;
+  }
+
+  // Set compression parameters according to pre-defined compression level table.
+  // Note that exact compression parameters are dynamically determined,
+  // depending on both compression level and source content size (when known).
+  // Value 0 means default, and default level is 3.
+  // Setting a level does not automatically set all other compression parameters
+  // to default. Setting this will however eventually dynamically impact the compression
+  // parameters which have not been manually set. The manually set
+  // ones will 'stick'.
+  google.protobuf.UInt32Value compression_level = 1;
+
+  // A 32-bits checksum of content is written at end of frame. If not set, defaults to false.
+  bool enable_checksum = 2;
+
+  // The higher the value of selected strategy, the more complex it is,
+  // resulting in stronger and slower compression.
+  // Special: value 0 means "use default strategy".
+  Strategy strategy = 3 [(validate.rules).enum = {defined_only: true}];
+
+  // Value for compressor's next output buffer. If not set, defaults to 4096.
+  google.protobuf.UInt32Value chunk_size = 5 [(validate.rules).uint32 = {lte: 65536 gte: 4096}];
+
+  // Enable QAT to accelerate zstd compression or not. If not set, defaults to false.
+  bool enable_qat_zstd = 6;
+
+  // Fallback to software for qatzstd when input size is less than this value.
+  // Valid only enable_qat_zstd is true. 0 means no fallback at all. If not set, defaults to 4000.
+  google.protobuf.UInt32Value qat_zstd_fallback_threshold = 7
+      [(validate.rules).uint32 = {lte: 65536 gte: 0}];
+}

api/versioning/BUILD

@@ -10,6 +10,7 @@ proto_library(
     visibility = ["//visibility:public"],
     deps = [
         "//contrib/envoy/extensions/compression/qatzip/compressor/v3alpha:pkg",
+        "//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg",
         "//contrib/envoy/extensions/config/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/checksum/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/http/dynamo/v3:pkg",

bazel/foreign_cc/qatzstd.patch

@@ -0,0 +1,85 @@
+diff --git a/src/Makefile b/src/Makefile
+index 1abf10d..a0c7e9a 100644
+--- a/src/Makefile
++++ b/src/Makefile
+@@ -41,6 +41,7 @@ LIBDIR     ?= $(INSTALLDIR)/lib
+ INCLUDEDIR ?= $(INSTALLDIR)/include
+
+ CP ?= cp
++MKDIR ?= mkdir
+
+ ENABLE_USDM_DRV ?= 0
+ ifneq ($(ICP_ROOT), )
+@@ -55,10 +56,8 @@ ifneq ($(ICP_ROOT), )
+ else
+ 	QATFLAGS = -DINTREE
+ 	LDFLAGS  = -lqat
+-	ifneq ($(ENABLE_USDM_DRV), 0)
+-		QATFLAGS += -DENABLE_USDM_DRV
+-		LDFLAGS	 += -lusdm
+-	endif
++	QATFLAGS += -DENABLE_USDM_DRV
++	LDFLAGS	 += -lusdm
+ endif
+
+ ifdef ZSTDLIB
+@@ -69,8 +68,8 @@ CFLAGS += -Wall -Werror -Wextra -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+ 		  -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes \
+ 		  -Wundef -Wpointer-arith -Wvla -Wformat=2 -Winit-self \
+ 		  -Wfloat-equal -Wwrite-strings -Wredundant-decls -Wc++-compat \
+-		  -pedantic -fstack-protector-strong -fPIE -fPIC \
+-		  -fno-delete-null-pointer-checks -fwrapv -fno-strict-overflow
++		  -pedantic -fstack-protector-strong \
++		  -fno-delete-null-pointer-checks -fwrapv 
+
+ DEBUGLEVEL ?=0
+
+@@ -81,27 +80,30 @@ else
+ 	QATFLAGS += -O3
+ endif
+
++$(info INSTALLDIR="$(INSTALLDIR)")
++$(info CPPFLAGS="$(CPPFLAGS)")
++
+ qatseqprod.o: qatseqprod.c
+-	$(CC) -c $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@
++	$(CC) -c $(CPPFLAGS) $(CFLAGS) $(QATFLAGS) $(DEBUGFLAGS) $^ -o $@
+
+ lib: qatseqprod.o
+ 	$(AR) rc libqatseqprod.a $^
+-	$(CC) -shared $^ $(LDFLAGS) -o libqatseqprod.so
++	@echo qatseqprod library successfully build
+
+ .PHONY: install
+ install: lib
++	$(MKDIR) -p $(LIBDIR)
++	$(MKDIR) -p $(INCLUDEDIR)
+ 	$(CP) libqatseqprod.a $(LIBDIR)
+-	$(CP) libqatseqprod.so $(LIBDIR)
+ 	$(CP) qatseqprod.h $(INCLUDEDIR)
+ 	@echo qatseqprod library successfully installed
+
+ .PHONY: uninstall
+ uninstall:
+ 	$(RM) $(LIBDIR)/libqatseqprod.a
+-	$(RM) $(LIBDIR)/libqatseqprod.so
+ 	$(RM) $(INCLUDEDIR)/qatseqprod.h
+ 	@echo qatseqprod library successfully uninstalled
+
+ clean:
+ 	$(RM) *.o
+-	$(RM) libqatseqprod.a libqatseqprod.so
++	$(RM) libqatseqprod.a 
+diff --git a/test/Makefile b/test/Makefile
+index dff0c8e..4ba01b2 100644
+--- a/test/Makefile
++++ b/test/Makefile
+@@ -34,7 +34,7 @@
+ # #######################################################################
+ LIB     = ../src
+
+-LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB)
++LDFLAGS = $(LIB)/libqatseqprod.a -I$(LIB) -L$(LIB) -l:libqatseqprod.a -l:libqat.a -l:libusdm.a -l:libzstd.a -lpthread -lcrypto
+
+ ifneq ($(ICP_ROOT), )
+ 	LDFLAGS += -lqat_s -lusdm_drv_s -Wl,-rpath,$(ICP_ROOT)/build -L$(ICP_ROOT)/build

bazel/repositories.bzl

@@ -313,6 +313,7 @@ def envoy_dependencies(skip_targets = []):
     _com_github_intel_ipp_crypto_crypto_mb_fips()
     _com_github_intel_qatlib()
     _com_github_intel_qatzip()
+    _com_github_qat_zstd()
     _com_github_lz4_lz4()
     _com_github_jbeder_yaml_cpp()
     _com_github_libevent_libevent()
@@ -583,6 +584,14 @@ def _com_github_intel_qatzip():
         build_file_content = BUILD_ALL_CONTENT,
     )
 
+def _com_github_qat_zstd():
+    external_http_archive(
+        name = "com_github_qat_zstd",
+        build_file_content = BUILD_ALL_CONTENT,
+        patch_args = ["-p1"],
+        patches = ["@envoy//bazel/foreign_cc:qatzstd.patch"],
+    )
+
 def _com_github_lz4_lz4():
     external_http_archive(
         name = "com_github_lz4_lz4",

bazel/repository_locations.bzl

@@ -446,7 +446,7 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         urls = ["https://github.com/intel/qatlib/archive/refs/tags/{version}.tar.gz"],
         use_category = ["dataplane_ext"],
         release_date = "2023-11-15",
-        extensions = ["envoy.tls.key_providers.qat", "envoy.compression.qatzip.compressor"],
+        extensions = ["envoy.tls.key_providers.qat", "envoy.compression.qatzip.compressor", "envoy.compression.qatzstd.compressor"],
         cpe = "N/A",
         license = "BSD-3-Clause",
         license_url = "https://github.com/intel/qatlib/blob/{version}/LICENSE",
@@ -466,6 +466,21 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         license = "BSD-3-Clause",
         license_url = "https://github.com/intel/QATzip/blob/v{version}/LICENSE",
     ),
+    com_github_qat_zstd = dict(
+        project_name = "QAT-ZSTD-Plugin",
+        project_desc = "Intel® QuickAssist Technology ZSTD Plugin (QAT ZSTD Plugin)",
+        project_url = "https://github.com/intel/QAT-ZSTD-Plugin/",
+        version = "0.1.0",
+        sha256 = "74c5bfbb3b0c6f1334e128ee0b43958d1d34751a4762e54e8f970c443e445f33",
+        strip_prefix = "QAT-ZSTD-Plugin-{version}",
+        urls = ["https://github.com/intel/QAT-ZSTD-Plugin/archive/refs/tags/v{version}.tar.gz"],
+        use_category = ["dataplane_ext"],
+        extensions = [
+            "envoy.compression.qatzstd.compressor",
+        ],
+        release_date = "2023-09-08",
+        cpe = "N/A",
+    ),
     com_github_luajit_luajit = dict(
         project_name = "LuaJIT",
         project_desc = "Just-In-Time compiler for Lua",

changelogs/current.yaml

@@ -172,6 +172,9 @@ new_features:
   change: |
     added support for :ref:`%UPSTREAM_CONNECTION_ID% <config_access_log_format_upstream_connection_id>` for the upstream connection
     identifier.
+- area: compression
+  change: |
+    Added qatzstd :ref:`compressor <envoy_v3_api_msg_extensions.compression.qatzstd.compressor.v3alpha.Qatzstd>`.
 - area: aws_lambda
   change: |
     Added :ref:`host_rewrite <envoy_v3_api_field_extensions.filters.http.aws_lambda.v3.Config.host_rewrite>` config to be used

contrib/all_contrib_extensions.bzl

@@ -18,6 +18,7 @@ ARM64_SKIP_CONTRIB_TARGETS = [
     "envoy.tls.key_providers.qat",
     "envoy.network.connection_balance.dlb",
     "envoy.compression.qatzip.compressor",
+    "envoy.compression.qatzstd.compressor",
 ]
 PPC_SKIP_CONTRIB_TARGETS = [
     "envoy.tls.key_providers.cryptomb",
@@ -26,6 +27,7 @@ PPC_SKIP_CONTRIB_TARGETS = [
     "envoy.network.connection_balance.dlb",
     "envoy.regex_engines.hyperscan",
     "envoy.compression.qatzip.compressor",
+    "envoy.compression.qatzstd.compressor",
 ]
 
 FIPS_LINUX_X86_SKIP_CONTRIB_TARGETS = [

contrib/contrib_build_config.bzl

@@ -5,6 +5,7 @@ CONTRIB_EXTENSIONS = {
     #
 
     "envoy.compression.qatzip.compressor":                      "//contrib/qat/compression/qatzip/compressor/source:config",
+    "envoy.compression.qatzstd.compressor":                      "//contrib/qat/compression/qatzstd/compressor/source:config",
 
     #
     # HTTP filters

contrib/extensions_metadata.yaml

@@ -18,6 +18,11 @@ envoy.compression.qatzip.compressor:
   - envoy.compression.compressor
   security_posture: robust_to_untrusted_downstream_and_upstream
   status: alpha
+envoy.compression.qatzstd.compressor:
+  categories:
+  - envoy.compression.compressor
+  security_posture: robust_to_untrusted_downstream_and_upstream
+  status: alpha
 envoy.filters.http.squash:
   categories:
   - envoy.filters.http

contrib/qat/compression/qatzstd/compressor/source/BUILD

@@ -0,0 +1,67 @@
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "make")
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_contrib_extension",
+    "envoy_cc_library",
+    "envoy_contrib_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_contrib_package()
+
+make(
+    name = "qat-zstd",
+    build_data = ["@com_github_qat_zstd//:all"],
+    env = select({
+        "//bazel:clang_build": {
+            "CFLAGS": "-Wno-error=unused-parameter",
+        },
+        "//conditions:default": {},
+    }),
+    includes = [],
+    lib_source = "@com_github_qat_zstd//:all",
+    out_static_libs = ["libqatseqprod.a"],
+    tags = ["skip_on_windows"],
+    target_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+    targets = [
+        "",
+        "install",
+    ],
+    deps = [
+        "//contrib/qat:qatlib",
+        "//external:zstd",
+    ],
+)
+
+envoy_cc_library(
+    name = "compressor_lib",
+    srcs = ["qatzstd_compressor_impl.cc"],
+    hdrs = ["qatzstd_compressor_impl.h"],
+    deps = [
+        ":qat-zstd",
+        "//envoy/compression/compressor:compressor_interface",
+        "//envoy/server:factory_context_interface",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/compression/zstd/common:zstd_base_lib",
+        "//source/common/compression/zstd/compressor:compressor_base",
+    ],
+)
+
+envoy_cc_contrib_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        ":compressor_lib",
+        ":qat-zstd",
+        "//envoy/event:dispatcher_interface",
+        "//envoy/thread_local:thread_local_interface",
+        "//source/common/http:headers_lib",
+        "//source/extensions/compression/common/compressor:compressor_factory_base_lib",
+        "@envoy_api//contrib/envoy/extensions/compression/qatzstd/compressor/v3alpha:pkg_cc_proto",
+    ],
+)