diff --git a/.gitignore b/.gitignore index 7ce1cbb9..93fca663 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *.out **/Chart.lock **/charts/*.tgz + +bazel-* +compile_commands.json diff --git a/proxy/.bazelrc b/proxy/.bazelrc new file mode 100644 index 00000000..15762ca1 --- /dev/null +++ b/proxy/.bazelrc @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Keep envoy.bazelrc up-to-date by run: +# curl -sSL https://raw.githubusercontent.com/istio/proxy/master/.bazelrc > envoy.bazelrc +import %workspace%/envoy.bazelrc + +# Build with Clang by default. +build --config=clang + +# Build with embedded V8-based WebAssembly runtime. +build --define wasm=v8 + +# Build Proxy-WASM plugins as native extensions. +build --copt -DNULL_PLUGIN + +# Build with Docker. +build:docker --action_env='PATH=/opt/llvm/bin:/opt/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' --host_action_env='PATH=/opt/llvm/bin:/opt/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin' +build:docker --action_env='LLVM_CONFIG=/opt/llvm/bin/llvm-config' --host_action_env='LLVM_CONFIG=/opt/llvm/bin/llvm-config' +build:docker --repo_env='LLVM_CONFIG=/opt/llvm/bin/llvm-config' +build:docker --linkopt='-L/opt/llvm/lib' +build:docker --linkopt='-Wl,-rpath,/opt/llvm/lib' diff --git a/proxy/.bazelversion b/proxy/.bazelversion new file mode 100644 index 00000000..f22d756d --- /dev/null +++ b/proxy/.bazelversion @@ -0,0 +1 @@ +6.5.0 diff --git a/proxy/.clang-format b/proxy/.clang-format new file mode 100644 index 00000000..5283618f --- /dev/null +++ b/proxy/.clang-format @@ -0,0 +1,17 @@ +--- +Language: Cpp +AccessModifierOffset: -2 +ColumnLimit: 100 +DerivePointerAlignment: false +PointerAlignment: Left +SortIncludes: false +TypenameMacros: ['STACK_OF'] +... + +--- +Language: Proto +ColumnLimit: 100 +SpacesInContainerLiterals: false +AllowShortFunctionsOnASingleLine: false +ReflowComments: false +... diff --git a/proxy/BUILD b/proxy/BUILD new file mode 100644 index 00000000..85c84db8 --- /dev/null +++ b/proxy/BUILD @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_binary", +) + +exports_files(["LICENSE"]) + +ISTIO_EXTENSIONS = [ + "@istio//source/extensions/common/workload_discovery:api_lib", # Experimental: WIP + "@istio//source/extensions/filters/http/alpn:config_lib", + "@istio//source/extensions/filters/http/istio_stats", + "@istio//source/extensions/filters/http/peer_metadata:filter_lib", + "@istio//source/extensions/filters/network/metadata_exchange:config_lib", +] + +OPEA_EXTENSIONS = [ + "//source/extensions/filters/http/guardrails:config", +] + +envoy_cc_binary( + name = "envoy", + repository = "@envoy", + deps = ISTIO_EXTENSIONS + OPEA_EXTENSIONS + [ + "@envoy//source/exe:envoy_main_entry_lib", + ], +) diff --git a/proxy/Makefile b/proxy/Makefile new file mode 100644 index 00000000..ffef6e50 --- /dev/null +++ b/proxy/Makefile @@ -0,0 +1,57 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +BUILD_IMG ?= envoy-build-ubuntu +BUILD_TAG ?= 75238004b0fcfd8a7f71d380d7a774dda5c39622 +ISTIO_IMG ?= proxyv2 +ISTIO_TAG ?= 1.23.0 +DOCKER_REGISTRY ?= docker.io/opea + +BUILD_WITH_CONTAINER ?= 0 +BAZEL_ARG ?= + +ifeq ($(BUILD_WITH_CONTAINER),0) + +.PHONY: build +build: + bazel build -c opt $(BAZEL_ARG) envoy + mkdir -p bin + cp -f bazel-bin/envoy bin/envoy + +.PHONY: test +test: + bazel test $(BAZEL_ARG) //test/... + +else + +.PHONY: build +build: + docker run -it \ + --rm \ + -v /tmp/proxy-docker-build:/root/.cache \ + -v $(shell pwd):/source \ + $(DOCKER_REGISTRY)/$(BUILD_IMG):$(BUILD_TAG) \ + "/bin/bash" "-c" "cd /source && export PATH=/opt/llvm/bin:$$PATH && BAZEL_ARG=\"--config=docker\" make" + +.PHONY: test +test: + docker run -it \ + --rm \ + -v /tmp/proxy-docker-build:/root/.cache \ + -v $(shell pwd):/source \ + $(DOCKER_REGISTRY)/$(BUILD_IMG):$(BUILD_TAG) \ + "/bin/bash" "-c" "cd /source && export PATH=/opt/llvm/bin:$$PATH && BAZEL_ARG=\"--config=docker\" make test" + +endif + +.PHONY: image +image: + docker build -f tools/Dockerfile -t $(DOCKER_REGISTRY)/$(ISTIO_IMG):$(ISTIO_TAG) + +.PHONY: build-image +build-image: + docker build -f tools/Dockerfile-build -t $(DOCKER_REGISTRY)/$(BUILD_IMG):$(BUILD_TAG) . + +.PHONY: compilation-database +compilation-database: + $(shell bazel info output_base)/external/envoy/tools/gen_compilation_database.py --vscode //source/... //test/... diff --git a/proxy/README.md b/proxy/README.md new file mode 100644 index 00000000..df558803 --- /dev/null +++ b/proxy/README.md @@ -0,0 +1,59 @@ +# OPEA Pipeline Proxy + +OPEA Pipeline Proxy is an enhancement of the default Istio proxy with additional features designed specifically for OPEA RAG pipelines. + +## Features + +- [Guardrails](./deployments/guardrails/README.md) + +## Build + +OPEA Pipeline Proxy is based on Istio proxy and Envoy, you can refer to [Building Envoy with Bazel](https://github.com/envoyproxy/envoy/blob/main/bazel/README.md) for build locally. In addition, Clang and OpenVINO is required to build OPEA Pipeline Proxy. + +```sh +# Build OPEA Pipeline Proxy. The binary will be generated to `bin/envoy`. +make + +# Build the image. The image will be tagged as `opea/proxyv2:` by default. +make image +``` + +You can also build OPEA Pipeline Proxy in the build container. + +```sh +# Build the build image. +make build-image + +# Build OPEA Pipeline Proxy with the build container. The binary will be generated to `bin/envoy`. +BUILD_WITH_CONTAINER=1 make + +# Build the image. The image will be tagged as `opea/proxyv2:` by default. +make image +``` + +## Deployment + +Before deploying OPEA Pipeline Proxy, you have to install Istio. Please follow the steps [here](https://istio.io/latest/docs/setup/install/istioctl/) for Istio installation. + +During the installation, you have to assign the OPEA Pipeline Proxy to deploy instead of the default one delivered by Istio. + +```sh +# Use the default Istio pilot and the proxyv2 delivered by OPEA. +istioctl install --set hub=docker.io/opea --set components.pilot.hub=docker.io/istio +``` + +You can also use the annotation [here](https://istio.io/latest/docs/reference/config/annotations/#SidecarProxyImage) to inject OPEA Pipeline Proxy as sidecars. + +## Development + +You can generate the [JSON Compilation Database](https://clang.llvm.org/docs/JSONCompilationDatabase.html) for Visual Studio Code with [clangd](https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.vscode-clangd) extension and other compatible tools. + +```sh +make compilation-database +``` + +You can test OPEA Pipeline Proxy with the following command. + +```sh +make test +``` diff --git a/proxy/WORKSPACE b/proxy/WORKSPACE new file mode 100644 index 00000000..a4e2f63d --- /dev/null +++ b/proxy/WORKSPACE @@ -0,0 +1,71 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +workspace(name = "dev_opea_proxy") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# Update Istio and Envoy by: +# 1. Determine SHA256 `wget https://github.com/istio/proxy/archive/$COMMIT.tar.gz && sha256sum $COMMIT.tar.gz`. +# 2. Update .bazelversion, envoy.bazelrc, BUILD, Makefile and tools/Dockerfile* if needed. +# +# Istio 1.23.0. +ISTIO_SHA = "165f7597596f5b4de7025bb635a7b59d1c3fe27e" +ISTIO_SHA256 = "ac44db76cd00442c0485f5ce99d69ae1044f33ea62c2dccb8e72c26bcf03b4e3" +ISTIO_ORG = "istio" +ISTIO_REPO = "proxy" + +http_archive( + name = "istio", + sha256 = ISTIO_SHA256, + strip_prefix = ISTIO_REPO + "-" + ISTIO_SHA, + url = "https://github.com/" + ISTIO_ORG + "/" + ISTIO_REPO + "/archive/" + ISTIO_SHA + ".tar.gz", +) + +# TODO(zhxie): remove external repository fetch since it is the same repository as Istio. +http_archive( + name = "envoy_build_config", + sha256 = ISTIO_SHA256, + strip_prefix = ISTIO_REPO + "-" + ISTIO_SHA + "/bazel/extension_config", + url = "https://github.com/" + ISTIO_ORG + "/" + ISTIO_REPO + "/archive/" + ISTIO_SHA + ".tar.gz", +) + +ENVOY_SHA = "97d97011f522006a816f8fe340c8a6f23d0dd712" +ENVOY_SHA256 = "0ed516f812f477b23da723aa379fbe96fcd84a04112d8a5e27bfcfb34c986eea" +ENVOY_ORG = "envoyproxy" +ENVOY_REPO = "envoy" + +http_archive( + name = "envoy", + sha256 = ENVOY_SHA256, + strip_prefix = ENVOY_REPO + "-" + ENVOY_SHA, + url = "https://github.com/" + ENVOY_ORG + "/" + ENVOY_REPO + "/archive/" + ENVOY_SHA + ".tar.gz", +) + +load("@envoy//bazel:api_binding.bzl", "envoy_api_binding") + +envoy_api_binding() + +load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies") + +envoy_api_dependencies() + +load("@envoy//bazel:repositories.bzl", "envoy_dependencies") + +envoy_dependencies() + +load("@envoy//bazel:repositories_extra.bzl", "envoy_dependencies_extra") + +envoy_dependencies_extra(ignore_root_user_error = True) + +load("@envoy//bazel:python_dependencies.bzl", "envoy_python_dependencies") + +envoy_python_dependencies() + +load("@base_pip3//:requirements.bzl", "install_deps") + +install_deps() + +load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports") + +envoy_dependency_imports() diff --git a/proxy/bazel/BUILD b/proxy/bazel/BUILD new file mode 100644 index 00000000..916f3a44 --- /dev/null +++ b/proxy/bazel/BUILD @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/proxy/bazel/get_workspace_status b/proxy/bazel/get_workspace_status new file mode 100755 index 00000000..dc880499 --- /dev/null +++ b/proxy/bazel/get_workspace_status @@ -0,0 +1,6 @@ +#! /bin/bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +echo "BUILD_CONFIG ${BUILD_CONFIG:-default}" diff --git a/proxy/bazel/platform_mappings b/proxy/bazel/platform_mappings new file mode 100644 index 00000000..060231b0 --- /dev/null +++ b/proxy/bazel/platform_mappings @@ -0,0 +1,36 @@ +flags: + --cpu=arm64-v8a + --crosstool_top=//external:android/crosstool + @envoy//bazel:android_aarch64 + + --cpu=armeabi-v7a + --crosstool_top=//external:android/crosstool + @envoy//bazel:android_armeabi + + --cpu=x86 + --crosstool_top=//external:android/crosstool + @envoy//bazel:android_x86 + + --cpu=x86_64 + --crosstool_top=//external:android/crosstool + @envoy//bazel:android_x86_64 + + --cpu=darwin_x86_64 + --apple_platform_type=macos + @envoy//bazel:macos_x86_64 + + --cpu=darwin_arm64 + --apple_platform_type=macos + @envoy//bazel:macos_arm64 + + --cpu=ios_x86_64 + --apple_platform_type=ios + @envoy//bazel:ios_x86_64_platform + + --cpu=ios_sim_arm64 + --apple_platform_type=ios + @envoy//bazel:ios_sim_arm64_platform + + --cpu=ios_arm64 + --apple_platform_type=ios + @envoy//bazel:ios_arm64_platform diff --git a/proxy/deployments/guardrails/README.md b/proxy/deployments/guardrails/README.md new file mode 100644 index 00000000..b34ca512 --- /dev/null +++ b/proxy/deployments/guardrails/README.md @@ -0,0 +1,214 @@ +# Guardrails + +Guardrails is a security feature designed specifically for OPEA RAG pipelines. Guardrails helps eliminate hallucination and toxicity, and build trust from end users. + +Guardrails in OPEA Pipeline Proxy supports both rule-based detection and AI-powered content safety. You can leverage rule-based detection to design keyword filter to avoid the leakage of PII (personal identifiable information) or choose a SPAM model to filter SPAMs out. + +## Architecture + +```mermaid +flowchart LR + subgraph Filter[HTTP Filter] + direction BT + Guardrails <--> OpenVINO + end + Model --> Filter + Re(Request / Response) --> Filter + Filter --> Continue(Continue) + Filter --> Stop(Stop) +``` + +## Deployment + +The tutorial introduces how to leverage Guardrails and filter out SPAMs in requests on the ingress gateway for a ChatQnA pipeline. + +### Preparing the model used in Guardrails + +OPEA Pipeline Proxy leverages models in OpenVINO IR format. You can convert different models into OpenVINO IR format following [these instructions](https://docs.openvino.ai/2024/openvino-workflow/model-preparation/convert-model-to-ir.html). + +A Transformer model is usually consists of a tokenizer and some Transformer blocks. In OpenVINO IR format, the tokenizer and blocks can be merged into a file. You can merge the tokenizer and the model following [these instructions](https://docs.openvino.ai/2024/notebooks/openvino-tokenizers-with-output.html#merge-tokenizer-into-a-model). + +In the tutorial, we use [Titeiiko/OTIS-Official-Spam-Model](https://huggingface.co/Titeiiko/OTIS-Official-Spam-Model) for SPAM filtering. + +```sh +# Convert HuggingFace model to OpenVINO IR. +optimum-cli export openvino --model Titeiiko/OTIS-Official-Spam-Model OTIS-Official-Spam-Model +# Convert the tokenizer to OpenVINO IR. +convert_tokenizer Titeiiko/OTIS-Official-Spam-Model -o ./OTIS-Official-Spam-Model +``` + +```python +from openvino import Core, save_model +from openvino_tokenizers import connect_models + +core = Core() +tokenizer = core.read_model("OTIS-Official-Spam-Model/openvino_tokenizer.xml") +model = core.read_model("OTIS-Official-Spam-Model/openvino_model.xml") +# Merge the model and its tokenizer. +combined_model = connect_models(tokenizer, model) +save_model(combined_model, "OTIS-Official-Spam-Model/combined.xml") +``` + +### Mounting models in the container + +To leverage models, you have to mount them to the OPEA Pipeline Proxy container. Please follow the field [here](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#KubernetesResourcesSpec) to mount the model into the ingress gateway pod. + +You can also use the annotation [here](https://istio.io/latest/docs/reference/config/annotations/#SidecarUserVolume) and [here](https://istio.io/latest/docs/reference/config/annotations/#SidecarUserVolumeMount) to mount volumes in sidecars. + +In the tutorial, we mount the SPAM model from the config map to `/model`. + +```sh +kubectl -n istio-system create configmap model --from-file=combined.bin=model.bin --from-file=combined.xml=model.xml +``` + +```yaml +# We derive the configuration from the default manifest. +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +metadata: + namespace: istio-system +spec: + hub: docker.io/opea + tag: 1.23.0 + + components: + base: + enabled: true + pilot: + enabled: true + hub: docker.io/istio + ingressGateways: + - name: istio-ingressgateway + enabled: true + k8s: + volumes: + - name: model + volumeSource: + configMap: + name: model + volumeMounts: + - name: model + mountPath: /model + readOnly: true + egressGateways: + - name: istio-egressgateway + enabled: false + + values: + defaultRevision: "" + global: + istioNamespace: istio-system + configValidation: true + gateways: + istio-ingressgateway: {} + istio-egressgateway: {} +``` + +```sh +istioctl install -f +``` + +### Configuring the ingress gateway + +Before deploying Guardrails, Istio ingress gateway should be setup first. Use the following YAMLs to configure a gateway on port 80 and accept HTTP requests on `/` to the pipeline router. + +```yaml +apiVersion: networking.istio.io/v1 +kind: Gateway +metadata: + name: chatqna-gateway + namespace: istio-system +spec: + selector: + istio: ingressgateway + servers: + - hosts: + - chatqna-service.com + port: + name: http + number: 80 + protocol: HTTP +``` + +```yaml +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + name: chatqna-virtual-service + namespace: istio-system +spec: + gateways: + - istio-system/chatqna-gateway + hosts: + - chatqna-service.com + http: + - match: + - uri: + prefix: / + route: + - destination: + host: router-service.chatqa.svc.cluster.local + port: + number: 8080 +``` + +The service is ready. We can then determine the ingress IP and port for validation. If you are evaluating OPEA Pipeline Proxy in a local environment without any load balancer, you can change the service type of the service `istio-ingressgateway` from `LoadBalancer` to `NodePort`. + +```sh +export INGRESS_HOST=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +export INGRESS_PORT=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].port}') +``` + +We can then validating the Guardrails with the following `curl` command. + +```sh +curl -H "Host:chatqna-service.com" -d "{\"text\": \"Free money\"}" "http://$INGRESS_HOST:$INGRESS_PORT/" +# Omitted +``` + +### Integrating Guardrails in the ingress gateway + +We can now integrate Guardrails in the ingress gateway using Envoy filter. We will insert Guardrails in `GATEWAY` context and match all incoming requests. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: guardrails + namespace: istio-system +spec: + workloadSelector: + labels: + istio: ingressgateway + configPatches: + - applyTo: HTTP_FILTER + match: + context: GATEWAY + listener: + portNumber: 8080 + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + subFilter: + name: envoy.filters.http.router + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.guardrails + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.guardrails.v3.Guardrails + model_path: /model/combined.xml + source: REQUEST + action: ALLOW +``` + +### Validating the result + +After integrating Guardrails, we can validate the result using `curl` again. + +```sh +curl -H "Host:chatqna-service.com" -d "{\"text\": \"Free money\"}" "http://$INGRESS_HOST:$INGRESS_PORT/" +# Access denied +``` + +As a result, the request has been rejected. diff --git a/proxy/envoy.bazelrc b/proxy/envoy.bazelrc new file mode 100644 index 00000000..90b9b112 --- /dev/null +++ b/proxy/envoy.bazelrc @@ -0,0 +1,566 @@ +# Envoy specific Bazel build/test options. + +# Bazel doesn't need more than 200MB of memory for local build based on memory profiling: +# https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling +# The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large +# enough to consume all memory constrained by cgroup in large host. +# Limiting JVM heapsize here to let it do GC more when approaching the limit to +# leave room for compiler/linker. +# The number 3G is chosen heuristically to both support large VM and small VM with RBE. +# Startup options cannot be selected via config. +startup --host_jvm_args=-Xmx3g + +fetch --color=yes +run --color=yes + +build --color=yes +build --jobs=HOST_CPUS-1 +build --workspace_status_command="bash bazel/get_workspace_status" +build --incompatible_strict_action_env +build --java_runtime_version=remotejdk_11 +build --tool_java_runtime_version=remotejdk_11 +build --platform_mappings=bazel/platform_mappings +# silence absl logspam. +build --copt=-DABSL_MIN_LOG_LEVEL=4 +build --define envoy_mobile_listener=enabled +build --experimental_repository_downloader_retries=2 +build --enable_platform_specific_config + +# Pass CC, CXX and LLVM_CONFIG variables from the environment. +# We assume they have stable values, so this won't cause action cache misses. +build --action_env=CC --host_action_env=CC +build --action_env=CXX --host_action_env=CXX +build --action_env=LLVM_CONFIG --host_action_env=LLVM_CONFIG +# Do not pass through PATH however. +# It tends to have machine-specific values, such as dynamically created temp folders. +# This would make it impossible to share remote action cache hits among machines. +# build --action_env=PATH --host_action_env=PATH +# To make our own CI green, we do need that flag on Windows though. +build:windows --action_env=PATH --host_action_env=PATH + +# Allow stamped caches to bust when local filesystem changes. +# Requires setting `BAZEL_VOLATILE_DIRTY` in the env. +build --action_env=BAZEL_VOLATILE_DIRTY --host_action_env=BAZEL_VOLATILE_DIRTY + +# Prevent stamped caches from busting (eg in PRs) +# Requires setting `BAZEL_FAKE_SCM_REVISION` in the env. +build --action_env=BAZEL_FAKE_SCM_REVISION --host_action_env=BAZEL_FAKE_SCM_REVISION + +build --test_summary=terse + +build:docs-ci --action_env=DOCS_RST_CHECK=1 --host_action_env=DOCS_RST_CHECK=1 + +# TODO(keith): Remove once these 2 are the default +build --incompatible_config_setting_private_default_visibility +build --incompatible_enforce_config_setting_visibility + +test --test_verbose_timeout_warnings +test --experimental_ui_max_stdouterr_bytes=11712829 #default 1048576 + +# Allow tags to influence execution requirements +common --experimental_allow_tags_propagation + +# Enable position independent code (this is the default on macOS and Windows) +# (Workaround for https://github.com/bazelbuild/rules_foreign_cc/issues/421) +build:linux --copt=-fdebug-types-section +build:linux --copt=-fPIC +build:linux --copt=-Wno-deprecated-declarations +build:linux --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 +build:linux --cxxopt=-fsized-deallocation --host_cxxopt=-fsized-deallocation +build:linux --conlyopt=-fexceptions +build:linux --fission=dbg,opt +build:linux --features=per_object_debug_info +build:linux --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a +build:linux --action_env=BAZEL_LINKOPTS=-lm + +# We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace. +build --define absl=1 + +# Disable ICU linking for googleurl. +build --@com_googlesource_googleurl//build_config:system_icu=0 + +# Common flags for sanitizers +build:sanitizer --define tcmalloc=disabled +build:sanitizer --linkopt -ldl + +# Common flags for Clang +build:clang --action_env=BAZEL_COMPILER=clang +build:clang --linkopt=-fuse-ld=lld +build:clang --action_env=CC=clang --host_action_env=CC=clang +build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ + +# Flags for Clang + PCH +build:clang-pch --spawn_strategy=local +build:clang-pch --define=ENVOY_CLANG_PCH=1 + +# Use gold linker for gcc compiler. +build:gcc --linkopt=-fuse-ld=gold +build:gcc --test_env=HEAPCHECK= +build:gcc --action_env=BAZEL_COMPILER=gcc +build:gcc --action_env=CC=gcc --action_env=CXX=g++ + +# Clang-tidy +# TODO(phlax): enable this, its throwing some errors as well as finding more issues +# build:clang-tidy --@envoy_toolshed//format/clang_tidy:executable=@envoy//tools/clang-tidy +build:clang-tidy --@envoy_toolshed//format/clang_tidy:config=//:clang_tidy_config +build:clang-tidy --aspects @envoy_toolshed//format/clang_tidy:clang_tidy.bzl%clang_tidy_aspect +build:clang-tidy --output_groups=report +build:clang-tidy --build_tag_filters=-notidy + +# Basic ASAN/UBSAN that works for gcc +build:asan --config=sanitizer +# ASAN install its signal handler, disable ours so the stacktrace will be printed by ASAN +build:asan --define signal_trace=disabled +build:asan --define ENVOY_CONFIG_ASAN=1 +build:asan --build_tag_filters=-no_san +build:asan --test_tag_filters=-no_san +build:asan --copt -fsanitize=address,undefined +build:asan --linkopt -fsanitize=address,undefined +# vptr and function sanitizer are enabled in clang-asan if it is set up via bazel/setup_clang.sh. +build:asan --copt -fno-sanitize=vptr,function +build:asan --linkopt -fno-sanitize=vptr,function +build:asan --copt -DADDRESS_SANITIZER=1 +build:asan --copt -DUNDEFINED_SANITIZER=1 +build:asan --copt -D__SANITIZE_ADDRESS__ +build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1 +build:asan --test_env=UBSAN_OPTIONS=halt_on_error=true:print_stacktrace=1 +build:asan --test_env=ASAN_SYMBOLIZER_PATH +# ASAN needs -O1 to get reasonable performance. +build:asan --copt -O1 +build:asan --copt -fno-optimize-sibling-calls + +# Clang ASAN/UBSAN +build:clang-asan-common --config=clang +build:clang-asan-common --config=asan +build:clang-asan-common --linkopt -fuse-ld=lld +build:clang-asan-common --linkopt --rtlib=compiler-rt +build:clang-asan-common --linkopt --unwindlib=libgcc + +build:clang-asan --config=clang-asan-common +build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone.a +build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx.a +build:clang-asan --action_env=ENVOY_UBSAN_VPTR=1 +build:clang-asan --copt=-fsanitize=vptr,function +build:clang-asan --linkopt=-fsanitize=vptr,function + +# macOS +build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 +build:macos --action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin +build:macos --host_action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin +build:macos --define tcmalloc=disabled + +# macOS ASAN/UBSAN +build:macos-asan --config=asan +# Workaround, see https://github.com/bazelbuild/bazel/issues/6932 +build:macos-asan --copt -Wno-macro-redefined +build:macos-asan --copt -D_FORTIFY_SOURCE=0 +# Workaround, see https://github.com/bazelbuild/bazel/issues/4341 +build:macos-asan --copt -DGRPC_BAZEL_BUILD +# Dynamic link cause issues like: `dyld: malformed mach-o: load commands size (59272) > 32768` +build:macos-asan --dynamic_mode=off + +# Clang TSAN +build:clang-tsan --action_env=ENVOY_TSAN=1 +build:clang-tsan --config=sanitizer +build:clang-tsan --define ENVOY_CONFIG_TSAN=1 +build:clang-tsan --copt -fsanitize=thread +build:clang-tsan --linkopt -fsanitize=thread +build:clang-tsan --linkopt -fuse-ld=lld +build:clang-tsan --copt -DTHREAD_SANITIZER=1 +build:clang-tsan --build_tag_filters=-no_san,-no_tsan +build:clang-tsan --test_tag_filters=-no_san,-no_tsan +# Needed due to https://github.com/libevent/libevent/issues/777 +build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE +# https://github.com/abseil/abseil-cpp/issues/760 +# https://github.com/google/sanitizers/issues/953 +build:clang-tsan --test_env="TSAN_OPTIONS=report_atomic_races=0" +build:clang-tsan --test_timeout=120,600,1500,4800 + +# Clang MSAN - this is the base config for remote-msan and docker-msan. To run this config without +# our build image, follow https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo +# with libc++ instruction and provide corresponding `--copt` and `--linkopt` as well. +build:clang-msan --action_env=ENVOY_MSAN=1 +build:clang-msan --config=sanitizer +build:clang-msan --build_tag_filters=-no_san +build:clang-msan --test_tag_filters=-no_san +build:clang-msan --define ENVOY_CONFIG_MSAN=1 +build:clang-msan --copt -fsanitize=memory +build:clang-msan --linkopt -fsanitize=memory +build:clang-msan --linkopt -fuse-ld=lld +build:clang-msan --copt -fsanitize-memory-track-origins=2 +build:clang-msan --copt -DMEMORY_SANITIZER=1 +build:clang-msan --test_env=MSAN_SYMBOLIZER_PATH +# MSAN needs -O1 to get reasonable performance. +build:clang-msan --copt -O1 +build:clang-msan --copt -fno-optimize-sibling-calls + +# Clang with libc++ +build:libc++ --config=clang +build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ +build:libc++ --action_env=LDFLAGS=-stdlib=libc++ +build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ +build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a +build:libc++ --action_env=BAZEL_LINKOPTS=-lm:-pthread +build:libc++ --define force_libcpp=enabled +build:clang-libc++ --config=libc++ + +build:libc++20 --config=libc++ +# gRPC has a lot of deprecated-enum-enum-conversion warning. Remove once it is addressed +build:libc++20 --copt=-Wno-error=deprecated-enum-enum-conversion + +# Optimize build for binary size reduction. +build:sizeopt -c opt --copt -Os + +# Test options +build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH + +# Coverage options +coverage --config=coverage +coverage --build_tests_only + +build:coverage --action_env=BAZEL_USE_LLVM_NATIVE_COVERAGE=1 +build:coverage --action_env=GCOV=llvm-profdata +build:coverage --copt=-DNDEBUG +# 1.5x original timeout + 300s for trace merger in all categories +build:coverage --test_timeout=390,750,1500,5700 +build:coverage --define=dynamic_link_tests=true +build:coverage --define=ENVOY_CONFIG_COVERAGE=1 +build:coverage --cxxopt="-DENVOY_CONFIG_COVERAGE=1" +build:coverage --test_env=HEAPCHECK= +build:coverage --combined_report=lcov +build:coverage --strategy=TestRunner=remote,sandboxed,local +build:coverage --strategy=CoverageReport=sandboxed,local +build:coverage --experimental_use_llvm_covmap +build:coverage --experimental_generate_llvm_lcov +build:coverage --experimental_split_coverage_postprocessing +build:coverage --experimental_fetch_all_coverage_outputs +build:coverage --collect_code_coverage +build:coverage --instrumentation_filter="^//source(?!/common/quic/platform)[/:],^//envoy[/:],^//contrib(?!/.*/test)[/:]" +build:coverage --remote_download_minimal +build:coverage --define=tcmalloc=gperftools +build:coverage --define=no_debug_info=1 +# `--no-relax` is required for coverage to not err with `relocation R_X86_64_REX_GOTPCRELX` +build:coverage --linkopt=-Wl,-s,--no-relax +build:coverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only + +build:test-coverage --test_arg="-l trace" +build:test-coverage --test_arg="--log-path /dev/null" +build:test-coverage --test_tag_filters=-nocoverage,-fuzz_target +build:fuzz-coverage --config=plain-fuzzer +build:fuzz-coverage --run_under=@envoy//bazel/coverage:fuzz_coverage_wrapper.sh +build:fuzz-coverage --test_tag_filters=-nocoverage + +build:cache-local --remote_cache=grpc://localhost:9092 + +# Remote execution: https://docs.bazel.build/versions/master/remote-execution.html +build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +build:rbe-toolchain-clang --config=rbe-toolchain +build:rbe-toolchain-clang --platforms=@envoy_build_tools//toolchains:rbe_linux_clang_platform +build:rbe-toolchain-clang --host_platform=@envoy_build_tools//toolchains:rbe_linux_clang_platform +build:rbe-toolchain-clang --crosstool_top=@envoy_build_tools//toolchains/configs/linux/clang/cc:toolchain +build:rbe-toolchain-clang --extra_toolchains=@envoy_build_tools//toolchains/configs/linux/clang/config:cc-toolchain +build:rbe-toolchain-clang --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/llvm/bin + +build:rbe-toolchain-clang-libc++ --config=rbe-toolchain +build:rbe-toolchain-clang-libc++ --platforms=@envoy_build_tools//toolchains:rbe_linux_clang_libcxx_platform +build:rbe-toolchain-clang-libc++ --host_platform=@envoy_build_tools//toolchains:rbe_linux_clang_libcxx_platform +build:rbe-toolchain-clang-libc++ --crosstool_top=@envoy_build_tools//toolchains/configs/linux/clang_libcxx/cc:toolchain +build:rbe-toolchain-clang-libc++ --extra_toolchains=@envoy_build_tools//toolchains/configs/linux/clang_libcxx/config:cc-toolchain +build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/llvm/bin +build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ +build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ +build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled + +# Do not inherit from "clang-asan" to avoid picking up flags from local clang.bazelrc. +build:rbe-toolchain-asan --config=asan +build:rbe-toolchain-asan --linkopt -fuse-ld=lld +build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1 +build:rbe-toolchain-asan --copt=-fsanitize=vptr,function +build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function +build:rbe-toolchain-asan --linkopt='-L/opt/llvm/lib/clang/14.0.0/lib/x86_64-unknown-linux-gnu' +build:rbe-toolchain-asan --linkopt=-l:libclang_rt.ubsan_standalone.a +build:rbe-toolchain-asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx.a + +build:rbe-toolchain-msan --linkopt=-L/opt/libcxx_msan/lib +build:rbe-toolchain-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib +build:rbe-toolchain-msan --config=clang-msan + +build:rbe-toolchain-tsan --linkopt=-L/opt/libcxx_tsan/lib +build:rbe-toolchain-tsan --linkopt=-Wl,-rpath,/opt/libcxx_tsan/lib +build:rbe-toolchain-tsan --config=clang-tsan + +build:rbe-toolchain-gcc --config=rbe-toolchain +build:rbe-toolchain-gcc --platforms=@envoy_build_tools//toolchains:rbe_linux_gcc_platform +build:rbe-toolchain-gcc --host_platform=@envoy_build_tools//toolchains:rbe_linux_gcc_platform +build:rbe-toolchain-gcc --crosstool_top=@envoy_build_tools//toolchains/configs/linux/gcc/cc:toolchain +build:rbe-toolchain-gcc --extra_toolchains=@envoy_build_tools//toolchains/configs/linux/gcc/config:cc-toolchain + +build:rbe-toolchain-msvc-cl --host_platform=@envoy_build_tools//toolchains:rbe_windows_msvc_cl_platform +build:rbe-toolchain-msvc-cl --platforms=@envoy_build_tools//toolchains:rbe_windows_msvc_cl_platform +build:rbe-toolchain-msvc-cl --crosstool_top=@envoy_build_tools//toolchains/configs/windows/msvc-cl/cc:toolchain +build:rbe-toolchain-msvc-cl --extra_toolchains=@envoy_build_tools//toolchains/configs/windows/msvc-cl/config:cc-toolchain + +build:rbe-toolchain-clang-cl --host_platform=@envoy_build_tools//toolchains:rbe_windows_clang_cl_platform +build:rbe-toolchain-clang-cl --platforms=@envoy_build_tools//toolchains:rbe_windows_clang_cl_platform +build:rbe-toolchain-clang-cl --crosstool_top=@envoy_build_tools//toolchains/configs/windows/clang-cl/cc:toolchain +build:rbe-toolchain-clang-cl --extra_toolchains=@envoy_build_tools//toolchains/configs/windows/clang-cl/config:cc-toolchain + +build:remote --spawn_strategy=remote,sandboxed,local +build:remote --strategy=Javac=remote,sandboxed,local +build:remote --strategy=Closure=remote,sandboxed,local +build:remote --strategy=Genrule=remote,sandboxed,local + +# Windows bazel does not allow sandboxed as a spawn strategy +build:remote-windows --spawn_strategy=remote,local +build:remote-windows --strategy=Javac=remote,local +build:remote-windows --strategy=Closure=remote,local +build:remote-windows --strategy=Genrule=remote,local +build:remote-windows --strategy=CppLink=local +build:remote-windows --remote_timeout=7200 +build:remote-windows --google_default_credentials=true +build:remote-windows --remote_download_toplevel + +build:remote-clang --config=remote +build:remote-clang --config=rbe-toolchain-clang + +build:remote-clang-libc++ --config=remote +build:remote-clang-libc++ --config=rbe-toolchain-clang-libc++ + +build:remote-gcc --config=remote +build:remote-gcc --config=gcc +build:remote-gcc --config=rbe-toolchain-gcc + +build:remote-asan --config=remote +build:remote-asan --config=rbe-toolchain-clang-libc++ +build:remote-asan --config=rbe-toolchain-asan + +build:remote-msan --config=remote +build:remote-msan --config=rbe-toolchain-clang-libc++ +build:remote-msan --config=rbe-toolchain-msan + +build:remote-tsan --config=remote +build:remote-tsan --config=rbe-toolchain-clang-libc++ +build:remote-tsan --config=rbe-toolchain-tsan + +build:remote-msvc-cl --config=remote-windows +build:remote-msvc-cl --config=msvc-cl +build:remote-msvc-cl --config=rbe-toolchain-msvc-cl + +build:remote-clang-cl --config=remote-windows +build:remote-clang-cl --config=clang-cl +build:remote-clang-cl --config=rbe-toolchain-clang-cl + +## Compile-time-options testing +# Right now, none of the available compile-time options conflict with each other. If this +# changes, this build type may need to be broken up. +build:compile-time-options --define=admin_html=disabled +build:compile-time-options --define=signal_trace=disabled +build:compile-time-options --define=hot_restart=disabled +build:compile-time-options --define=google_grpc=disabled +build:compile-time-options --define=boringssl=fips +build:compile-time-options --define=log_debug_assert_in_release=enabled +build:compile-time-options --define=path_normalization_by_default=true +build:compile-time-options --define=deprecated_features=disabled +build:compile-time-options --define=tcmalloc=gperftools +build:compile-time-options --define=zlib=ng +build:compile-time-options --define=uhv=enabled +build:compile-time-options --config=libc++20 +build:compile-time-options --test_env=ENVOY_HAS_EXTRA_EXTENSIONS=true +build:compile-time-options --@envoy//bazel:http3=False +build:compile-time-options --@envoy//source/extensions/filters/http/kill_request:enabled + +# Docker sandbox +# NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:f94a38f62220a2b017878b790b6ea98a0f6c5f9c@sha256:2dd96b6f43c08ccabd5f4747fce5854f5f96af509b32e5cf6493f136e9833649 +build:docker-sandbox --spawn_strategy=docker +build:docker-sandbox --strategy=Javac=docker +build:docker-sandbox --strategy=Closure=docker +build:docker-sandbox --strategy=Genrule=docker +build:docker-sandbox --define=EXECUTOR=remote +build:docker-sandbox --experimental_docker_verbose +build:docker-sandbox --experimental_enable_docker_sandbox + +build:docker-clang --config=docker-sandbox +build:docker-clang --config=rbe-toolchain-clang + +build:docker-clang-libc++ --config=docker-sandbox +build:docker-clang-libc++ --config=rbe-toolchain-clang-libc++ + +build:docker-gcc --config=docker-sandbox +build:docker-gcc --config=rbe-toolchain-gcc + +build:docker-asan --config=docker-sandbox +build:docker-asan --config=rbe-toolchain-clang-libc++ +build:docker-asan --config=rbe-toolchain-asan + +build:docker-msan --config=docker-sandbox +build:docker-msan --config=rbe-toolchain-clang-libc++ +build:docker-msan --config=rbe-toolchain-msan + +build:docker-tsan --config=docker-sandbox +build:docker-tsan --config=rbe-toolchain-clang-libc++ +build:docker-tsan --config=rbe-toolchain-tsan + +# CI configurations +build:remote-ci --config=ci +build:remote-ci --remote_download_minimal + +# Note this config is used by mobile CI also. +build:ci --noshow_progress +build:ci --noshow_loading_progress +build:ci --test_output=errors + +# Fuzz builds + +# Shared fuzzing configuration. +build:fuzzing --define=ENVOY_CONFIG_ASAN=1 +build:fuzzing --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +build:fuzzing --config=libc++ + +# Fuzzing without ASAN. This is useful for profiling fuzzers without any ASAN artifacts. +build:plain-fuzzer --config=fuzzing +build:plain-fuzzer --define=FUZZING_ENGINE=libfuzzer +# The fuzzing rules provide their own instrumentation, but it is currently +# disabled due to bazelbuild/bazel#12888. Instead, we provide instrumentation at +# the top level through these options. +build:plain-fuzzer --copt=-fsanitize=fuzzer-no-link +build:plain-fuzzer --linkopt=-fsanitize=fuzzer-no-link + +build:asan-fuzzer --config=plain-fuzzer +build:asan-fuzzer --config=clang-asan +build:asan-fuzzer --copt=-fno-omit-frame-pointer +# Remove UBSAN halt_on_error to avoid crashing on protobuf errors. +build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 + +build:oss-fuzz --config=fuzzing +build:oss-fuzz --define=FUZZING_ENGINE=oss-fuzz +build:oss-fuzz --@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz +build:oss-fuzz --@rules_fuzzing//fuzzing:cc_engine_sanitizer=none +build:oss-fuzz --dynamic_mode=off +build:oss-fuzz --strip=never +build:oss-fuzz --copt=-fno-sanitize=vptr +build:oss-fuzz --linkopt=-fno-sanitize=vptr +build:oss-fuzz --define=tcmalloc=disabled +build:oss-fuzz --define=signal_trace=disabled +build:oss-fuzz --copt=-D_LIBCPP_DISABLE_DEPRECATION_WARNINGS +build:oss-fuzz --define=force_libcpp=enabled +build:oss-fuzz --linkopt=-lc++ +build:oss-fuzz --linkopt=-pthread + +# Compile database generation config +build:compdb --build_tag_filters=-nocompdb + +# Windows build quirks +build:windows --action_env=TMPDIR +build:windows --define signal_trace=disabled +build:windows --define hot_restart=disabled +build:windows --define tcmalloc=disabled +build:windows --define wasm=disabled +build:windows --define manual_stamp=manual_stamp +build:windows --cxxopt="/std:c++20" +build:windows --output_groups=+pdb_file + +# TODO(wrowe,sunjayBhatia): Resolve bugs upstream in curl and rules_foreign_cc +# See issue https://github.com/bazelbuild/rules_foreign_cc/issues/301 +build:windows --copt="-DCARES_STATICLIB" +build:windows --copt="-DNGHTTP2_STATICLIB" +build:windows --copt="-DCURL_STATICLIB" + +# Override any clang preference if building msvc-cl +# Drop the determinism feature (-DDATE etc are a no-op in msvc-cl) +build:msvc-cl --action_env=USE_CLANG_CL="" +build:msvc-cl --define clang_cl=0 +build:msvc-cl --features=-determinism + +# Windows build behaviors when using clang-cl +build:clang-cl --action_env=USE_CLANG_CL=1 +build:clang-cl --define clang_cl=1 + +# Required to work around Windows clang-cl build defects +# Ignore conflicting definitions of _WIN32_WINNT +# Override determinism flags (DATE etc) is valid on clang-cl compiler +build:clang-cl --copt="-Wno-macro-redefined" +build:clang-cl --copt="-Wno-builtin-macro-redefined" +# Workaround problematic missing override declarations of mocks +# TODO: resolve this class of problematic mocks, e.g. +# ./test/mocks/http/stream.h(16,21): error: 'addCallbacks' +# overrides a member function but is not marked 'override' +# MOCK_METHOD(void, addCallbacks, (StreamCallbacks & callbacks)); +build:clang-cl --copt="-Wno-inconsistent-missing-override" + +# Defaults to 'auto' - Off for windows, so override to linux behavior +build:windows --enable_runfiles=yes + +# This should become adopted by bazel as the default +build:windows --features=compiler_param_file + +# These options attempt to force a monolithic binary including the CRT +build:windows --features=fully_static_link +build:windows --features=static_link_msvcrt +build:windows --dynamic_mode=off + +# RBE (Google) +build:cache-google --google_default_credentials=true +build:cache-google --remote_cache=grpcs://remotebuildexecution.googleapis.com +build:cache-google --remote_instance_name=projects/envoy-ci/instances/default_instance +build:cache-google --remote_timeout=7200 +build:rbe-google --remote_executor=grpcs://remotebuildexecution.googleapis.com +build:rbe-google --config=cache-google + +build:rbe-google-bes --bes_backend=grpcs://buildeventservice.googleapis.com +build:rbe-google-bes --bes_results_url=https://source.cloud.google.com/results/invocations/ +build:rbe-google-bes --bes_upload_mode=fully_async + +# RBE (Engflow mobile) +build:rbe-engflow --google_default_credentials=false +build:rbe-engflow --remote_cache=grpcs://envoy.cluster.engflow.com +build:rbe-engflow --remote_executor=grpcs://envoy.cluster.engflow.com +build:rbe-engflow --bes_backend=grpcs://envoy.cluster.engflow.com/ +build:rbe-engflow --bes_results_url=https://envoy.cluster.engflow.com/invocation/ +build:rbe-engflow --credential_helper=*.engflow.com=%workspace%/bazel/engflow-bazel-credential-helper.sh +build:rbe-engflow --grpc_keepalive_time=30s +build:rbe-engflow --remote_timeout=3600s +build:rbe-engflow --bes_timeout=3600s +build:rbe-engflow --bes_upload_mode=fully_async +build:rbe-engflow --nolegacy_important_outputs + +build:cache-envoy-engflow --google_default_credentials=false +build:cache-envoy-engflow --remote_cache=grpcs://morganite.cluster.engflow.com +build:cache-envoy-engflow --remote_timeout=3600s +build:cache-envoy-engflow --credential_helper=*.engflow.com=%workspace%/bazel/engflow-bazel-credential-helper.sh +build:cache-envoy-engflow --grpc_keepalive_time=30s +build:bes-envoy-engflow --bes_backend=grpcs://morganite.cluster.engflow.com/ +build:bes-envoy-engflow --bes_results_url=https://morganite.cluster.engflow.com/invocation/ +build:bes-envoy-engflow --bes_timeout=3600s +build:bes-envoy-engflow --bes_upload_mode=fully_async +build:rbe-envoy-engflow --config=cache-envoy-engflow +build:rbe-envoy-engflow --config=bes-envoy-engflow +build:rbe-envoy-engflow --remote_executor=grpcs://morganite.cluster.engflow.com +build:rbe-envoy-engflow --remote_default_exec_properties=container-image=docker://docker.io/envoyproxy/envoy-build-ubuntu:f94a38f62220a2b017878b790b6ea98a0f6c5f9c@sha256:2dd96b6f43c08ccabd5f4747fce5854f5f96af509b32e5cf6493f136e9833649 + +############################################################################# +# debug: Various Bazel debugging flags +############################################################################# +# debug/bazel +common:debug-bazel --announce_rc +common:debug-bazel -s +# debug/sandbox +common:debug-sandbox --verbose_failures +common:debug-sandbox --sandbox_debug +# debug/coverage +common:debug-coverage --action_env=VERBOSE_COVERAGE=true +common:debug-coverage --test_env=VERBOSE_COVERAGE=true +common:debug-coverage --test_env=DISPLAY_LCOV_CMD=true +common:debug-coverage --config=debug-tests +# debug/tests +common:debug-tests --test_output=all +# debug/everything +common:debug --config=debug-bazel +common:debug --config=debug-sandbox +common:debug --config=debug-coverage +common:debug --config=debug-tests + +try-import %workspace%/clang.bazelrc +try-import %workspace%/user.bazelrc +try-import %workspace%/local_tsan.bazelrc diff --git a/proxy/source/extensions/common/inference/BUILD b/proxy/source/extensions/common/inference/BUILD new file mode 100644 index 00000000..4352466f --- /dev/null +++ b/proxy/source/extensions/common/inference/BUILD @@ -0,0 +1,31 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_library", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +envoy_cc_library( + name = "inference_runtime_interface", + hdrs = ["inference_runtime.h"], + repository = "@envoy", + deps = ["@envoy//envoy/common:pure_lib"], + alwayslink = 1, +) + +envoy_cc_library( + name = "utility_lib", + srcs = ["utility.cc"], + hdrs = ["utility.h"], + repository = "@envoy", + deps = [ + ":inference_runtime_interface", + "//source/extensions/inference_runtime/openvino:config", + "@envoy//envoy/server:factory_context_interface", + ], +) diff --git a/proxy/source/extensions/common/inference/inference_runtime.h b/proxy/source/extensions/common/inference/inference_runtime.h new file mode 100644 index 00000000..88712216 --- /dev/null +++ b/proxy/source/extensions/common/inference/inference_runtime.h @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "envoy/common/pure.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Inference { + +class InferenceSession { +public: + virtual ~InferenceSession() = default; + + virtual bool classify(const std::string& input) PURE; +}; + +using InferenceSessionPtr = std::unique_ptr; + +class InferenceRuntime { +public: + virtual ~InferenceRuntime() = default; + + virtual InferenceSessionPtr createInferenceSession() PURE; +}; + +using InferenceRuntimePtr = std::unique_ptr; + +} // namespace Inference +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/common/inference/utility.cc b/proxy/source/extensions/common/inference/utility.cc new file mode 100644 index 00000000..db1b16c6 --- /dev/null +++ b/proxy/source/extensions/common/inference/utility.cc @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "source/extensions/common/inference/utility.h" + +#include "source/extensions/inference_runtime/openvino/config.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Inference { + +InferenceRuntimePtr createInferenceRuntime(const std::string& model_path, double threshold, + Server::Configuration::FactoryContext& context) { + InferenceRuntimePtr runtime = + std::make_unique( + model_path, threshold, context.serverFactoryContext().threadLocal()); + return runtime; +} + +} // namespace Inference +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/common/inference/utility.h b/proxy/source/extensions/common/inference/utility.h new file mode 100644 index 00000000..6d74f0ad --- /dev/null +++ b/proxy/source/extensions/common/inference/utility.h @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "envoy/server/factory_context.h" + +#include "source/extensions/common/inference/inference_runtime.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Inference { + +InferenceRuntimePtr createInferenceRuntime(const std::string& model_path, double threshold, + Server::Configuration::FactoryContext& context); + +} // namespace Inference +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/filters/http/guardrails/BUILD b/proxy/source/extensions/filters/http/guardrails/BUILD new file mode 100644 index 00000000..1f616bdf --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/BUILD @@ -0,0 +1,47 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_library", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +envoy_cc_library( + name = "guardrails_lib", + srcs = ["filter.cc"], + hdrs = ["filter.h"], + repository = "@envoy", + deps = [ + "//source/extensions/common/inference:utility_lib", + "@envoy//envoy/server:filter_config_interface", + "@envoy//source/common/buffer:buffer_lib", + "@envoy//source/common/json:json_loader_lib", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + repository = "@envoy", + deps = [ + ":config_cc_proto", + ":guardrails_lib", + "@envoy//envoy/registry", + "@envoy//source/extensions/filters/http/common:factory_base_lib", + ], +) + +cc_proto_library( + name = "config_cc_proto", + deps = ["proto"], +) + +proto_library( + name = "proto", + srcs = ["config.proto"], +) diff --git a/proxy/source/extensions/filters/http/guardrails/config.cc b/proxy/source/extensions/filters/http/guardrails/config.cc new file mode 100644 index 00000000..89df5c66 --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/config.cc @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "source/extensions/filters/http/guardrails/config.h" + +#include "source/extensions/common/inference/utility.h" +#include "source/extensions/filters/http/guardrails/filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Guardrails { + +absl::StatusOr +FilterConfigFactory::createFilterFactoryFromProto(const Protobuf::Message& config, + const std::string&, + Server::Configuration::FactoryContext& context) { + return createFilterFactory( + dynamic_cast(config), + context); +} + +ProtobufTypes::MessagePtr FilterConfigFactory::createEmptyConfigProto() { + return ProtobufTypes::MessagePtr{ + new envoy::extensions::filters::http::guardrails::v3::Guardrails}; +} + +Http::FilterFactoryCb FilterConfigFactory::createFilterFactory( + const envoy::extensions::filters::http::guardrails::v3::Guardrails& proto_config, + Server::Configuration::FactoryContext& context) { + Extensions::Common::Inference::InferenceRuntimePtr runtime = + Extensions::Common::Inference::createInferenceRuntime(proto_config.model_path(), + proto_config.threshold(), context); + Source source; + switch (proto_config.source()) { + case envoy::extensions::filters::http::guardrails::v3::Guardrails::REQUEST: + source = Source::Request; + break; + case envoy::extensions::filters::http::guardrails::v3::Guardrails::RESPONSE: + source = Source::Response; + break; + default: + PANIC_DUE_TO_CORRUPT_ENUM; + } + Action action; + switch (proto_config.action()) { + case envoy::extensions::filters::http::guardrails::v3::Guardrails::ALLOW: + action = Action::Allow; + break; + case envoy::extensions::filters::http::guardrails::v3::Guardrails::DENY: + action = Action::Deny; + break; + default: + PANIC_DUE_TO_CORRUPT_ENUM; + } + + FilterConfigSharedPtr filter_config{ + std::make_shared(std::move(runtime), source, action)}; + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_unique(filter_config)); + }; +} + +REGISTER_FACTORY(FilterConfigFactory, Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace Guardrails +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/filters/http/guardrails/config.h b/proxy/source/extensions/filters/http/guardrails/config.h new file mode 100644 index 00000000..52fedb08 --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/config.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "source/extensions/filters/http/common/factory_base.h" +#include "source/extensions/filters/http/guardrails/config.pb.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Guardrails { + +class FilterConfigFactory : public Server::Configuration::NamedHttpFilterConfigFactory { +public: + absl::StatusOr + createFilterFactoryFromProto(const Protobuf::Message& config, const std::string& stat_prefix, + Server::Configuration::FactoryContext& context) override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + std::string name() const override { return "envoy.filters.http.guardrails"; } + +private: + Http::FilterFactoryCb + createFilterFactory(const envoy::extensions::filters::http::guardrails::v3::Guardrails& config_pb, + Server::Configuration::FactoryContext& context); +}; + +} // namespace Guardrails +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/filters/http/guardrails/config.proto b/proxy/source/extensions/filters/http/guardrails/config.proto new file mode 100644 index 00000000..624429ca --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/config.proto @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +syntax = "proto3"; + +package envoy.extensions.filters.http.guardrails.v3; + +option java_package = "io.envoyproxy.envoy.extensions.filters.http.guardrails.v3"; +option java_outer_classname = "GuardrailsProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/guardrails/v3;guardrailsv3"; + +message Guardrails { + enum Source { + REQUEST = 0; + + RESPONSE = 1; + } + + enum Action { + ALLOW = 0; + + DENY = 1; + } + + string model_path = 1; + + double threshold = 2; + + Source source = 3; + + Action action = 4; +} diff --git a/proxy/source/extensions/filters/http/guardrails/filter.cc b/proxy/source/extensions/filters/http/guardrails/filter.cc new file mode 100644 index 00000000..754afd21 --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/filter.cc @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "source/extensions/filters/http/guardrails/filter.h" + +#include "source/common/json/json_loader.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Guardrails { + +namespace { +std::string getText(const std::string& buffer) { + absl::StatusOr loader = Json::Factory::loadFromStringNoThrow(buffer); + if (!loader.ok()) { + return ""; + } + + absl::StatusOr value_or_error; + value_or_error = loader.value()->getValue("text"); + if (!value_or_error.ok() || !absl::holds_alternative(value_or_error.value())) { + return ""; + } + + return absl::get(value_or_error.value()); +} +} // namespace + +void Filter::onDestroy() {} + +Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap&, bool end_stream) { + if (config_->source() == Source::Response || end_stream) { + return Http::FilterHeadersStatus::Continue; + } + return Http::FilterHeadersStatus::StopIteration; +} + +Http::FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_stream) { + if (config_->source() == Source::Response) { + return Http::FilterDataStatus::Continue; + } + + if (end_stream) { + decoder_callbacks_->addDecodedData(data, true); + + std::string content = decoder_callbacks_->decodingBuffer()->toString(); + bool matched = session_->classify(getText(content)); + if ((config_->action() == Action::Allow && matched) || + (config_->action() == Action::Deny && !matched)) { + return Http::FilterDataStatus::Continue; + } + + decoder_callbacks_->sendLocalReply(Http::Code::Forbidden, "Access denied", nullptr, + absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + return Http::FilterDataStatus::StopIterationAndBuffer; +} + +Http::FilterTrailersStatus Filter::decodeTrailers(Http::RequestTrailerMap&) { + return Http::FilterTrailersStatus::Continue; +} + +void Filter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) { + decoder_callbacks_ = &callbacks; +} + +Http::Filter1xxHeadersStatus Filter::encode1xxHeaders(Http::ResponseHeaderMap&) { + return Http::Filter1xxHeadersStatus::Continue; +} + +Http::FilterHeadersStatus Filter::encodeHeaders(Http::ResponseHeaderMap&, bool end_stream) { + if (config_->source() == Source::Request || end_stream) { + return Http::FilterHeadersStatus::Continue; + } + return Http::FilterHeadersStatus::StopIteration; +} + +Http::FilterDataStatus Filter::encodeData(Buffer::Instance& data, bool end_stream) { + if (config_->source() == Source::Request) { + return Http::FilterDataStatus::Continue; + } + + if (end_stream) { + encoder_callbacks_->addEncodedData(data, true); + + std::string content = encoder_callbacks_->encodingBuffer()->toString(); + bool matched = session_->classify(getText(content)); + if ((config_->action() == Action::Allow && matched) || + (config_->action() == Action::Deny && !matched)) { + return Http::FilterDataStatus::Continue; + } + + encoder_callbacks_->sendLocalReply(Http::Code::InternalServerError, "Access denied", nullptr, + absl::nullopt, ""); + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + return Http::FilterDataStatus::StopIterationAndBuffer; +} + +Http::FilterTrailersStatus Filter::encodeTrailers(Http::ResponseTrailerMap&) { + return Http::FilterTrailersStatus::Continue; +} + +Http::FilterMetadataStatus Filter::encodeMetadata(Http::MetadataMap&) { + return Http::FilterMetadataStatus::Continue; +} + +void Filter::setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) { + encoder_callbacks_ = &callbacks; +} + +} // namespace Guardrails +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/filters/http/guardrails/filter.h b/proxy/source/extensions/filters/http/guardrails/filter.h new file mode 100644 index 00000000..6e97fd17 --- /dev/null +++ b/proxy/source/extensions/filters/http/guardrails/filter.h @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "envoy/http/filter.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/extensions/common/inference/inference_runtime.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Guardrails { + +enum class Source { Request, Response }; + +enum class Action { Allow, Deny }; + +class FilterConfig { +public: + FilterConfig(Extensions::Common::Inference::InferenceRuntimePtr runtime, Source source, + Action action) + : runtime_(std::move(runtime)), source_(source), action_(action) {} + + Extensions::Common::Inference::InferenceSessionPtr createInferenceSession() const { + return runtime_->createInferenceSession(); + } + Source source() const { return source_; } + Action action() const { return action_; } + +protected: + Extensions::Common::Inference::InferenceRuntimePtr runtime_; + Source source_; + Action action_; +}; + +using FilterConfigSharedPtr = std::shared_ptr; + +class Filter : public Http::StreamFilter, Logger::Loggable { +public: + Filter(FilterConfigSharedPtr config) + : config_(config), session_(config_->createInferenceSession()) {} + + // Http::StreamFilterBase + void onDestroy() override; + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::RequestHeaderMap& headers, + bool end_stream) override; + Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus decodeTrailers(Http::RequestTrailerMap& trailers) override; + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override; + + // Http::StreamEncoderFilter + Http::Filter1xxHeadersStatus encode1xxHeaders(Http::ResponseHeaderMap& headers) override; + Http::FilterHeadersStatus encodeHeaders(Http::ResponseHeaderMap& headers, + bool end_stream) override; + Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override; + Http::FilterTrailersStatus encodeTrailers(Http::ResponseTrailerMap& trailers) override; + Http::FilterMetadataStatus encodeMetadata(Http::MetadataMap& metadata_map) override; + void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) override; + +protected: + FilterConfigSharedPtr config_; + Extensions::Common::Inference::InferenceSessionPtr session_; + Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; + Http::StreamEncoderFilterCallbacks* encoder_callbacks_{}; +}; + +} // namespace Guardrails +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/inference_runtime/openvino/BUILD b/proxy/source/extensions/inference_runtime/openvino/BUILD new file mode 100644 index 00000000..e24f35e3 --- /dev/null +++ b/proxy/source/extensions/inference_runtime/openvino/BUILD @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_library", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + repository = "@envoy", + linkopts = [ + "-lopenvino", + "-lopenvino_c", + "-lopenvino_ir_frontend", + ], + deps = [ + "//source/extensions/common/inference:inference_runtime_interface", + "@envoy//envoy/thread_local:thread_local_interface", + ], +) diff --git a/proxy/source/extensions/inference_runtime/openvino/config.cc b/proxy/source/extensions/inference_runtime/openvino/config.cc new file mode 100644 index 00000000..217c5728 --- /dev/null +++ b/proxy/source/extensions/inference_runtime/openvino/config.cc @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "source/extensions/inference_runtime/openvino/config.h" + +namespace Envoy { +namespace Extensions { +namespace InferenceRuntime { +namespace OpenVino { + +CompiledModelThreadLocal::CompiledModelThreadLocal(ov::Core core, + std::shared_ptr model) { + TRY_ASSERT_MAIN_THREAD { compiled_model_ = core.compile_model(model, "CPU"); } + END_TRY CATCH(const ov::Exception& e, + { throw EnvoyException(fmt::format("Failed to compile model: {}", e.what())); }); +} + +InferenceRuntime::InferenceRuntime(const std::string& model_path, double threshold, + ThreadLocal::SlotAllocator& tls) + : threshold_(threshold), + tls_(ThreadLocal::TypedSlot::makeUnique(tls)) { + // Add OpenVINO Tokenizers extension. + TRY_ASSERT_MAIN_THREAD { core_.add_extension("/usr/lib/libopenvino_tokenizers.so"); } + END_TRY CATCH(const ov::Exception& e, { + throw EnvoyException(fmt::format("Failed to add OpenVINO Tokenizers: {}", e.what())); + }); + TRY_ASSERT_MAIN_THREAD { model_ = core_.read_model(model_path); } + END_TRY CATCH(const ov::Exception& e, + { throw EnvoyException(fmt::format("Failed to read model: {}", e.what())); }); + RELEASE_ASSERT(model_ != nullptr, "Invalid model"); + + tls_->set([this](Event::Dispatcher&) { + return std::make_shared(core_, model_); + }); +} + +Common::Inference::InferenceSessionPtr InferenceRuntime::createInferenceSession() { + return std::make_unique(this); +} + +InferenceSession::InferenceSession(InferenceRuntime* runtime) : runtime_(runtime) {} + +bool InferenceSession::classify(const std::string& input) { + ov::InferRequest request = runtime_->compiledModel().create_infer_request(); + std::string& i = const_cast(input); + ov::Tensor tensor(ov::element::string, ov::Shape{1}, &i); + request.set_input_tensor(tensor); + + request.start_async(); + request.wait(); + + ov::Tensor output = request.get_output_tensor(); + const float* result = output.data(); + ENVOY_LOG(debug, "Inference output: {}", *result); + + return *result > runtime_->threshold(); +} + +} // namespace OpenVino +} // namespace InferenceRuntime +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/source/extensions/inference_runtime/openvino/config.h b/proxy/source/extensions/inference_runtime/openvino/config.h new file mode 100644 index 00000000..e3eec7f8 --- /dev/null +++ b/proxy/source/extensions/inference_runtime/openvino/config.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "envoy/thread_local/thread_local.h" + +#include "source/extensions/common/inference/inference_runtime.h" + +#include "openvino/openvino.hpp" + +namespace Envoy { +namespace Extensions { +namespace InferenceRuntime { +namespace OpenVino { + +struct CompiledModelThreadLocal : public ThreadLocal::ThreadLocalObject { + CompiledModelThreadLocal(ov::Core core, std::shared_ptr model); + + ov::CompiledModel compiled_model_; +}; + +using CompiledModelThreadLocalPtr = std::unique_ptr; + +class InferenceRuntime : public Common::Inference::InferenceRuntime { +public: + InferenceRuntime(const std::string& model_path, double threshold, + ThreadLocal::SlotAllocator& tls); + + Common::Inference::InferenceSessionPtr createInferenceSession() override; + double threshold() const { return threshold_; } + ov::CompiledModel& compiledModel() { return tls_->get()->compiled_model_; } + +protected: + ov::Core core_; + std::shared_ptr model_; + double threshold_; + ThreadLocal::TypedSlotPtr tls_; +}; + +class InferenceSession : public Common::Inference::InferenceSession, + Logger::Loggable { +public: + InferenceSession(InferenceRuntime* runtime); + + bool classify(const std::string& input) override; + +protected: + InferenceRuntime* runtime_; +}; + +} // namespace OpenVino +} // namespace InferenceRuntime +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/test/extensions/filters/http/guardrails/BUILD b/proxy/test/extensions/filters/http/guardrails/BUILD new file mode 100644 index 00000000..b888851a --- /dev/null +++ b/proxy/test/extensions/filters/http/guardrails/BUILD @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +load( + "@envoy//bazel:envoy_build_system.bzl", + "envoy_cc_test", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +envoy_cc_test( + name = "filter_test", + srcs = ["filter_test.cc"], + data = [ + "//test/extensions/filters/http/guardrails/test_data:model", + ], + repository = "@envoy", + deps = [ + "//source/extensions/filters/http/guardrails:guardrails_lib", + "@envoy//test/mocks/server:factory_context_mocks", + "@envoy//test/test_common:utility_lib", + ], +) diff --git a/proxy/test/extensions/filters/http/guardrails/filter_test.cc b/proxy/test/extensions/filters/http/guardrails/filter_test.cc new file mode 100644 index 00000000..f5f3b3c4 --- /dev/null +++ b/proxy/test/extensions/filters/http/guardrails/filter_test.cc @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2024 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "source/extensions/common/inference/utility.h" +#include "source/extensions/filters/http/guardrails/filter.h" + +#include "test/mocks/server/factory_context.h" +#include "test/test_common/environment.h" +#include "test/test_common/utility.h" + +using testing::_; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Guardrails { + +static constexpr absl::string_view ModelPath = + "/test/extensions/filters/http/guardrails/test_data/model.xml"; + +std::string GetModelPath() { + return TestEnvironment::runfilesDirectory("dev_opea_proxy") + std::string(ModelPath); +} + +class FilterTest : public testing::Test { +public: + void setup(std::string& model_path, double threshold, Source source, Action action) { + EXPECT_CALL(server_factory_context_, threadLocal()).WillOnce(ReturnRef(tls_)); + EXPECT_CALL(context_, serverFactoryContext()).WillOnce(ReturnRef(server_factory_context_)); + Common::Inference::InferenceRuntimePtr runtime = + Common::Inference::createInferenceRuntime(model_path, threshold, context_); + config_ = std::make_shared(std::move(runtime), source, action); + filter_ = std::make_shared(config_); + + filter_->setDecoderFilterCallbacks(decoded_callbacks_); + filter_->setEncoderFilterCallbacks(encoded_callbacks_); + } + +protected: + NiceMock tls_; + Server::Configuration::MockServerFactoryContext server_factory_context_; + Server::Configuration::MockFactoryContext context_; + FilterConfigSharedPtr config_; + std::shared_ptr filter_; + NiceMock decoded_callbacks_; + NiceMock encoded_callbacks_; +}; + +TEST_F(FilterTest, RequestAllow) { + std::string path = GetModelPath(); + setup(path, 0, Source::Request, Action::Allow); + + Buffer::InstancePtr buf(new Buffer::OwnedImpl()); + EXPECT_CALL(decoded_callbacks_, addDecodedData(_, _)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { buf->add(data); })); + EXPECT_CALL(decoded_callbacks_, decodingBuffer()).WillRepeatedly(Return(buf.get())); + + Buffer::OwnedImpl data("{\"text\": \"What a beautiful world!\"}"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, true)); + buf->drain(buf->length()); + Buffer::OwnedImpl data2("{\"text\": \"Free money!\"}"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data2, true)); +} + +TEST_F(FilterTest, RequestDeny) { + std::string path = GetModelPath(); + setup(path, 0, Source::Request, Action::Deny); + + Buffer::InstancePtr buf(new Buffer::OwnedImpl()); + EXPECT_CALL(decoded_callbacks_, addDecodedData(_, _)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { buf->add(data); })); + EXPECT_CALL(decoded_callbacks_, decodingBuffer()).WillRepeatedly(Return(buf.get())); + + Buffer::OwnedImpl data("{\"text\": \"What a beautiful world!\"}"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data, true)); + buf->drain(buf->length()); + Buffer::OwnedImpl data2("{\"text\": \"Free money!\"}"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data2, true)); +} + +TEST_F(FilterTest, ResponseAllow) { + std::string path = GetModelPath(); + setup(path, 0, Source::Response, Action::Allow); + + Buffer::InstancePtr buf(new Buffer::OwnedImpl()); + EXPECT_CALL(encoded_callbacks_, addEncodedData(_, _)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { buf->add(data); })); + EXPECT_CALL(encoded_callbacks_, encodingBuffer()).WillRepeatedly(Return(buf.get())); + + Buffer::OwnedImpl data("{\"text\": \"What a beautiful world!\"}"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data, true)); + buf->drain(buf->length()); + Buffer::OwnedImpl data2("{\"text\": \"Free money!\"}"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data2, true)); +} + +TEST_F(FilterTest, ResponseDeny) { + std::string path = GetModelPath(); + setup(path, 0, Source::Response, Action::Deny); + + Buffer::InstancePtr buf(new Buffer::OwnedImpl()); + EXPECT_CALL(encoded_callbacks_, addEncodedData(_, _)) + .WillRepeatedly(Invoke([&](Buffer::Instance& data, bool) { buf->add(data); })); + EXPECT_CALL(encoded_callbacks_, encodingBuffer()).WillRepeatedly(Return(buf.get())); + + Buffer::OwnedImpl data("{\"text\": \"What a beautiful world!\"}"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data, true)); + buf->drain(buf->length()); + Buffer::OwnedImpl data2("{\"text\": \"Free money!\"}"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data2, true)); +} + +} // namespace Guardrails +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/proxy/test/extensions/filters/http/guardrails/test_data/BUILD b/proxy/test/extensions/filters/http/guardrails/test_data/BUILD new file mode 100644 index 00000000..532a7b11 --- /dev/null +++ b/proxy/test/extensions/filters/http/guardrails/test_data/BUILD @@ -0,0 +1,14 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "model", + srcs = [ + "model.bin", + "model.xml", + ], +) diff --git a/proxy/test/extensions/filters/http/guardrails/test_data/model.bin b/proxy/test/extensions/filters/http/guardrails/test_data/model.bin new file mode 100644 index 00000000..e7389550 Binary files /dev/null and b/proxy/test/extensions/filters/http/guardrails/test_data/model.bin differ diff --git a/proxy/test/extensions/filters/http/guardrails/test_data/model.xml b/proxy/test/extensions/filters/http/guardrails/test_data/model.xml new file mode 100644 index 00000000..799e8c61 --- /dev/null +++ b/proxy/test/extensions/filters/http/guardrails/test_data/model.xml @@ -0,0 +1,4903 @@ + + + + + + + + -1 + + + + + + + + 30522 + 128 + + + + + + + + + + + 30522 + 128 + + + + + 30522 + 128 + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + -1 + + + + + -1 + + + -1 + + + -1 + + + + + + + + 43 + + + + + + + + 1 + + + + + + + -1 + + + -1 + + + -1 + + + 43 + + + 1 + + + + + -1 + + + -1 + + + -1 + + + + + + + + -1 + + + -1 + + + -1 + + + + + -1 + + + -1 + + + -1 + + + + + + + + 6 + + + + + + + + 0 + + + + + + + -1 + + + -1 + + + -1 + + + 6 + + + 0 + + + + + -1 + + + -1 + + + -1 + + + + + + + -1 + + + -1 + + + -1 + + + + + -1 + + + -1 + + + -1 + + + + + + + + -1 + + + + + 1 + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + -1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + + + + + + + + 3 + + + + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + 3 + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + + + + + + 202 + + + + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + 202 + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + + + + + + 323082 + + + + + + + + 323082 + + + + + -1 + + + -1 + + + -1 + + + + + + + + + + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + + + + -1 + + + -1 + + + -1 + + + + + + + + -1 + + + -1 + + + + + -1 + + + + + + + + + + + + + + -1 + + + + + + -1 + + + + + + + + -1 + + + -1 + + + + + -1 + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + 3 + + + + + + + + + 1 + + + -1 + + + -1 + + + -1 + + + + + 1 + + + 3 + + + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + -1 + + + + + + + + -1 + + + -1 + + + + + -1 + + + + + + + + + + + + + + -1 + + + + + + + + + + + + + + + + + -1 + + + -1 + + + -1 + + + + + + + -1 + -1 + + + -1 + -1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + + + + + + + 30522 + 128 + + + -1 + -1 + + + + + + -1 + -1 + 128 + + + + + + + + 2 + 128 + + + + + + + + + + + 2 + 128 + + + + + 2 + 128 + + + + + + + + + + + + + -1 + + + -1 + + + -1 + + + + + + + -1 + -1 + + + -1 + -1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + + + + + + + 2 + 128 + + + -1 + -1 + + + + + + -1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + -1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 512 + 128 + + + + + + + + + + + 512 + 128 + + + + + 512 + 128 + + + + + + + + 1 + 512 + + + + + + + + + + + 2 + + + + + + + + + + + 2 + + + + + + + + + + + 1 + + + + + + + + -1 + -1 + + + + + 2 + + + + + + + + + + + + + + + + + + + + + + + + + + 2 + + + + + + + + + + + + + + + + + + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + 2 + + + 1 + + + 1 + + + 1 + + + + + 2 + + + + + + + + + + + 2 + + + + + + + + 1 + 512 + + + 2 + + + 2 + + + 2 + + + + + 1 + -1 + + + + + + + + 1 + -1 + + + + + 1 + -1 + + + + + + + + + + + + + + 512 + 128 + + + 1 + -1 + + + + + + 1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + -1 + 2 + -1 + 64 + + + -1 + 2 + -1 + 64 + + + + + -1 + 2 + -1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + + + + + 1 + 1 + 1 + 1 + + + + + 1 + 1 + 1 + 1 + + + + + + + + -1 + 2 + -1 + -1 + + + 1 + 1 + 1 + 1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + + + + + 1 + 1 + 1 + 1 + + + + + 1 + 1 + 1 + 1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + -1 + -1 + + + + + -1 + -1 + + + + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + + -1 + -1 + + + 1 + + + 1 + + + 1 + + + + + -1 + -1 + + + + + + + -1 + -1 + + + + + + -1 + 1 + -1 + + + + + + + + + + + + + -1 + 1 + -1 + + + + + + -1 + 1 + 1 + -1 + + + + + + + + + + + 4 + + + + + + + + + + + 4 + + + + + + + + + + + 4 + + + + + + + + -1 + 1 + 1 + -1 + + + 4 + + + 4 + + + 4 + + + + + -1 + 1 + 1 + -1 + + + + + + + + -1 + 1 + 1 + -1 + + + + + -1 + 1 + 1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + + + + + 1 + 1 + 1 + 1 + + + + + 1 + 1 + 1 + 1 + + + + + + + + -1 + 1 + 1 + -1 + + + 1 + 1 + 1 + 1 + + + + + -1 + 1 + 1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + -1 + 1 + 1 + -1 + + + + + -1 + 1 + 1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + + -1 + 1 + 1 + -1 + + + 1 + 1 + 1 + 1 + + + + + -1 + 1 + 1 + -1 + + + + + + + + -1 + 2 + -1 + -1 + + + -1 + 1 + 1 + -1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + -1 + 2 + -1 + -1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + -1 + 2 + -1 + -1 + + + -1 + 2 + -1 + 64 + + + + + -1 + 2 + -1 + 64 + + + + + + + + 4 + + + + + + + -1 + 2 + -1 + 64 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + + + + 3 + + + + + + + + -1 + -1 + 2 + 64 + + + 3 + + + + + -1 + -1 + 128 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + -1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 512 + 128 + + + + + + + + + + + 512 + 128 + + + + + 512 + 128 + + + + + + + + -1 + -1 + 128 + + + 512 + 128 + + + + + -1 + -1 + 512 + + + + + + + + 1 + 1 + 512 + + + + + + + + + + + 1 + 1 + 512 + + + + + 1 + 1 + 512 + + + + + + + + -1 + -1 + 512 + + + 1 + 1 + 512 + + + + + -1 + -1 + 512 + + + + + + + + -1 + -1 + 512 + + + + + -1 + -1 + 512 + + + + + + + + 128 + 512 + + + + + + + + + + + 128 + 512 + + + + + 128 + 512 + + + + + + + + -1 + -1 + 512 + + + 128 + 512 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + -1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + -1 + 2 + -1 + 64 + + + -1 + 2 + -1 + 64 + + + + + -1 + 2 + -1 + -1 + + + + + + + + 1 + 1 + 1 + 1 + + + + + + + + + + + 1 + 1 + 1 + 1 + + + + + 1 + 1 + 1 + 1 + + + + + + + + -1 + 2 + -1 + -1 + + + 1 + 1 + 1 + 1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + -1 + 2 + -1 + -1 + + + -1 + 1 + 1 + -1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + -1 + 2 + -1 + -1 + + + + + -1 + 2 + -1 + -1 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 4 + + + + + + + + -1 + -1 + 128 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + 4 + + + + + + + -1 + -1 + 2 + 64 + + + 4 + + + + + -1 + 2 + -1 + 64 + + + + + + + + -1 + 2 + -1 + -1 + + + -1 + 2 + -1 + 64 + + + + + -1 + 2 + -1 + 64 + + + + + + + + 4 + + + + + + + -1 + 2 + -1 + 64 + + + 4 + + + + + -1 + -1 + 2 + 64 + + + + + + + + + + + 3 + + + + + + + + -1 + -1 + 2 + 64 + + + 3 + + + + + -1 + -1 + 128 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + -1 + 128 + + + 128 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + -1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 512 + 128 + + + + + + + + + + + 512 + 128 + + + + + 512 + 128 + + + + + + + + -1 + -1 + 128 + + + 512 + 128 + + + + + -1 + -1 + 512 + + + + + + + + 1 + 1 + 512 + + + + + + + + + + + 1 + 1 + 512 + + + + + 1 + 1 + 512 + + + + + + + + -1 + -1 + 512 + + + 1 + 1 + 512 + + + + + -1 + -1 + 512 + + + + + + + + -1 + -1 + 512 + + + + + -1 + -1 + 512 + + + + + + + + 128 + 512 + + + + + + + + + + + 128 + 512 + + + + + 128 + 512 + + + + + + + + -1 + -1 + 512 + + + 128 + 512 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + -1 + -1 + 128 + + + -1 + -1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + 1 + 1 + 128 + + + + + + + + + + + 1 + 1 + 128 + + + + + 1 + 1 + 128 + + + + + + + + -1 + -1 + 128 + + + 1 + 1 + 128 + + + + + -1 + -1 + 128 + + + + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + + + + + 1 + + + + + + + + -1 + -1 + 128 + + + 1 + + + 1 + + + 1 + + + + + -1 + -1 + 128 + + + + + + + + + + + + + + -1 + -1 + 128 + + + + + + + -1 + 128 + + + + + + + + 128 + 128 + + + + + + + + + + + 128 + 128 + + + + + 128 + 128 + + + + + + + + -1 + 128 + + + 128 + 128 + + + + + -1 + 128 + + + + + + + + 1 + 128 + + + + + + + + + + + 1 + 128 + + + + + 1 + 128 + + + + + + + + -1 + 128 + + + 1 + 128 + + + + + -1 + 128 + + + + + + + -1 + 128 + + + + + -1 + 128 + + + + + + + + 2 + 128 + + + + + + + + + + + 2 + 128 + + + + + 2 + 128 + + + + + + + + -1 + 128 + + + 2 + 128 + + + + + -1 + 2 + + + + + + + + 1 + 2 + + + + + + + + + + + 1 + 2 + + + + + 1 + 2 + + + + + + + + -1 + 2 + + + 1 + 2 + + + + + -1 + 2 + + + + + + + -1 + 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proxy/tools/Dockerfile b/proxy/tools/Dockerfile new file mode 100644 index 00000000..c8c7169d --- /dev/null +++ b/proxy/tools/Dockerfile @@ -0,0 +1,25 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +ARG BUILD_TAG=75238004b0fcfd8a7f71d380d7a774dda5c39622 +ARG ISTIO_TAG=1.23.0 + +FROM opea/envoy-build-ubuntu:${BUILD_TAG} AS build + +FROM istio/proxyv2:${ISTIO_TAG} + +# Install libtbb. +RUN apt update && apt install -y libtbb12 + +# Install OpenVINO. +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libopenvino.so* /usr/lib +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libopenvino_c.so* /usr/lib +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libopenvino_ir_frontend.so* /usr/lib +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libopenvino_intel_cpu_plugin.so /usr/lib + +# Install OpenVINO Tokenizers. +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libcore_tokenizers.so /usr/lib +COPY --from=build --chown=0:0 --chmod=644 /usr/lib/libopenvino_tokenizers.so /usr/lib + +# Install Envoy. +COPY bin/envoy /usr/local/bin/envoy diff --git a/proxy/tools/Dockerfile-build b/proxy/tools/Dockerfile-build new file mode 100644 index 00000000..64f101c8 --- /dev/null +++ b/proxy/tools/Dockerfile-build @@ -0,0 +1,27 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +ARG BUILD_TAG=75238004b0fcfd8a7f71d380d7a774dda5c39622 + +FROM envoyproxy/envoy-build-ubuntu:${BUILD_TAG} + +# Install OpenVINO. +ARG OPENVINO_VERSION=2024.0 \ + OPENVINO_FULL_VERSION=2024.0.0.14509 \ + OPENVINO_SHA=34caeefd078 \ + OPENVINO_LIB_VERSION=2400 +RUN wget https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION}/linux/l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64.tgz && \ + tar -xf l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64.tgz && \ + rm l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64.tgz && \ + cp -r l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64/runtime/include/openvino /usr/include/openvino && \ + cp -r l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64/runtime/lib/intel64/* /usr/lib && \ + rm -r l_openvino_toolkit_ubuntu22_${OPENVINO_FULL_VERSION}.${OPENVINO_SHA}_x86_64 +RUN ln -s /usr/lib/libopenvino_ir_frontend.so.${OPENVINO_LIB_VERSION} /usr/lib/libopenvino_ir_frontend.so + +# Install OpenVINO Tokenizers. +ARG OPENVINO_TOKENIZERS_VERSION=0.0 +RUN wget https://storage.openvinotoolkit.org/repositories/openvino_tokenizers/packages/${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}/openvino_tokenizers_ubuntu22_${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}_x86_64.tgz && \ + tar -xf openvino_tokenizers_ubuntu22_${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}_x86_64.tgz && \ + rm openvino_tokenizers_ubuntu22_${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}_x86_64.tgz && \ + cp openvino_tokenizers_ubuntu22_${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}_x86_64/* /usr/lib && \ + rm -r openvino_tokenizers_ubuntu22_${OPENVINO_VERSION}.${OPENVINO_TOKENIZERS_VERSION}_x86_64