Add support for Nginx DOS feature

This change adds support for the Nginx DOS module. It includes custom resources, examples and documentation. Co-authored-by: Tomer Pasman <[email protected]> Co-authored-by: Tomer <=>
nginx · Dec 3, 2021 · 7b4bd01 · 7b4bd01
1 parent f46c7cd
commit 7b4bd01
Show file tree

Hide file tree

Showing 172 changed files with 10,038 additions and 1,020 deletions.
diff --git a/Makefile b/Makefile
@@ -45,7 +45,7 @@ update-codegen: ## Generate code
 
 .PHONY: update-crds
 update-crds: ## Update CRDs
-	go run sigs.k8s.io/controller-tools/cmd/controller-gen crd:crdVersions=v1 schemapatch:manifests=./deployments/common/crds/ paths=./pkg/apis/configuration/... output:dir=./deployments/common/crds
+	go run sigs.k8s.io/controller-tools/cmd/controller-gen crd:crdVersions=v1 schemapatch:manifests=./deployments/common/crds/ paths=./pkg/apis/.../... output:dir=./deployments/common/crds
 	@cp -Rp deployments/common/crds/ deployments/helm-chart/crds
 
 .PHONY: certificate-and-key
@@ -97,6 +97,14 @@ debian-image-plus: build ## Create Docker image for Ingress Controller (Debian w
 debian-image-nap-plus: build ## Create Docker image for Ingress Controller (Debian with NGINX Plus and App Protect WAF)
 	$(DOCKER_CMD) $(PLUS_ARGS) --build-arg BUILD_OS=debian-plus-nap --build-arg DEBIAN_VERSION=buster-slim
 
+.PHONY: debian-image-dos-plus
+debian-image-dos-plus: build ## Create Docker image for Ingress Controller (nginx plus with nap-dos)
+	$(DOCKER_CMD) $(PLUS_ARGS) --build-arg BUILD_OS=debian-plus-dos
+
+.PHONY: debian-image-nap-dos-plus
+debian-image-nap-dos-plus: build ## Create Docker image for Ingress Controller (nginx plus with nap and nap-dos)
+	$(DOCKER_CMD) $(PLUS_ARGS) --build-arg BUILD_OS=debian-plus-nap-dos --build-arg FILES=nap-common
+
 .PHONY: openshift-image
 openshift-image: build ## Create Docker image for Ingress Controller (UBI)
 	$(DOCKER_CMD) --build-arg BUILD_OS=ubi
@@ -113,6 +121,14 @@ openshift-image-nap-plus: build ## Create Docker image for Ingress Controller (U
 alpine-image-opentracing: build ## Create Docker image for Ingress Controller (Alpine with OpenTracing)
 	$(DOCKER_CMD) --build-arg BUILD_OS=alpine-opentracing
 
+.PHONY: openshift-image-dos-plus
+openshift-image-dos-plus: build ## Create Docker image for Ingress Controller (ubi with plus and dos)
+	$(DOCKER_CMD) $(PLUS_ARGS) $(NAP_ARGS) --secret id=rhel_license,src=rhel_license --build-arg BUILD_OS=ubi-plus-dos --build-arg UBI_VERSION=7
+
+.PHONY: openshift-image-nap-dos-plus
+openshift-image-nap-dos-plus: build ## Create Docker image for Ingress Controller (ubi with plus, nap and dos)
+	$(DOCKER_CMD) $(PLUS_ARGS) $(NAP_ARGS) --secret id=rhel_license,src=rhel_license --build-arg BUILD_OS=ubi-plus-nap-dos --build-arg UBI_VERSION=7
+
 .PHONY: debian-image-opentracing
 debian-image-opentracing: build ## Create Docker image for Ingress Controller (Debian with OpenTracing)
 	$(DOCKER_CMD) --build-arg BUILD_OS=opentracing
@@ -122,7 +138,7 @@ debian-image-opentracing-plus: build ## Create Docker image for Ingress Controll
 	$(DOCKER_CMD) $(PLUS_ARGS) --build-arg BUILD_OS=opentracing-plus
 
 .PHONY: all-images ## Create all the Docker images for Ingress Controller
-all-images: alpine-image alpine-image-plus debian-image debian-image-plus debian-image-nap-plus debian-image-opentracing debian-image-opentracing-plus openshift-image openshift-image-plus openshift-image-nap-plus
+all-images: alpine-image alpine-image-plus debian-image debian-image-plus debian-image-nap-plus debian-image-dos-plus debian-image-nap-dos-plus debian-image-opentracing debian-image-opentracing-plus openshift-image openshift-image-plus openshift-image-nap-plus
 
 .PHONY: push
 push: ## Docker push to PREFIX and TAG

diff --git a/build/Dockerfile b/build/Dockerfile
@@ -81,6 +81,37 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 # COPY build/*.crt  /usr/local/share/ca-certificates/
 # RUN update-ca-certificates
 
+############################################# Base image for Debian with NGINX Plus and App Protect Dos #############################################
+FROM debian-plus as debian-plus-dos
+ARG IC_VERSION
+ARG NGINX_PLUS_VERSION
+
+RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode=0644 \
+	--mount=type=secret,id=nginx-repo.key,dst=/etc/ssl/nginx/nginx-repo.key,mode=0644 \
+	set -x \
+	&& apt-get update \
+	&& apt-get -y install lsb-release ca-certificates \
+	&& printf "deb https://pkgs.nginx.com/app-protect-dos/${NGINX_PLUS_VERSION^^}/debian `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect-dos.list \
+	&& apt-get update \
+	&& apt-get -y install app-protect-dos \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& rm /etc/apt/sources.list.d/nginx-app-protect-dos.list
+
+############################################# Base image for Debian with NGINX, App Protect and App Protect Dos #############################################
+FROM debian-plus-nap as debian-plus-nap-dos
+ARG IC_VERSION
+ARG NGINX_PLUS_VERSION
+
+RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode=0644 \
+	--mount=type=secret,id=nginx-repo.key,dst=/etc/ssl/nginx/nginx-repo.key,mode=0644 \
+	set -x \
+	&& apt-get update \
+	&& apt-get -y install lsb-release ca-certificates \
+	&& printf "deb https://pkgs.nginx.com/app-protect-dos/debian `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect-dos.list \
+	&& apt-get update \
+	&& apt-get -y install app-protect-dos \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& rm /etc/apt/sources.list.d/nginx-app-protect-dos.list
 
 ############################################# Base image for UBI 8 #############################################
 FROM redhat/ubi8-minimal AS ubi-base-8
@@ -162,6 +193,41 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 # COPY build/*.crt  /etc/pki/ca-trust/source/anchors/
 # RUN update-ca-trust extract
 
+############################################# Base image for UBI with NGINX Plus and App Protect Dos #############################################
+FROM ubi-plus as ubi-plus-dos
+ARG NGINX_PLUS_VERSION
+
+RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode=0644 \
+	--mount=type=secret,id=nginx-repo.key,dst=/etc/ssl/nginx/nginx-repo.key,mode=0644 \
+	--mount=type=secret,id=rhel_license,dst=/tmp/rhel_license,mode=0644 \
+	source /tmp/rhel_license \
+	&& subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \
+	&& subscription-manager attach \
+    && curl -sS https://cs.nginx.com/static/files/app-protect-dos-7.repo > /etc/yum.repos.d/app-protect-dos-7.repo \
+    && subscription-manager repos --enable rhel-7-server-optional-rpms --enable rhel-7-server-extras-rpms \
+    && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \
+    && yum clean all \
+    && yum -y install epel-release \
+    && yum -y install app-protect-dos-${NGINX_PLUS_VERSION#r}* \
+    && rm /etc/yum.repos.d/app-protect-dos-7.repo \
+    && subscription-manager unregister
+
+############################################# Base image for UBI with NGINX Plus and App Protect and App Protect Dos #############################################
+FROM ubi-plus-nap as ubi-plus-nap-dos
+ARG NGINX_PLUS_VERSION
+
+RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode=0644 \
+	--mount=type=secret,id=nginx-repo.key,dst=/etc/ssl/nginx/nginx-repo.key,mode=0644 \
+	--mount=type=secret,id=rhel_license,dst=/tmp/rhel_license,mode=0644 \
+	source /tmp/rhel_license \
+	&& subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \
+	&& subscription-manager attach \
+    && curl -sS https://cs.nginx.com/static/files/app-protect-dos-7.repo > /etc/yum.repos.d/app-protect-dos-7.repo \
+    && yum clean all \
+    && yum -y install epel-release \
+    && yum -y install app-protect-dos-${NGINX_PLUS_VERSION#r}* \
+    && rm /etc/yum.repos.d/app-protect-dos-7.repo \
+    && subscription-manager unregister
 
 ############################################# Base images containing libs for Opentracing #############################################
 FROM opentracing/nginx-opentracing:nginx-1.21.4 as opentracing-lib
@@ -225,6 +291,9 @@ RUN --mount=target=/tmp [ -n "${BUILD_OS##*nap*}" ] && exit 0; mkdir -p /etc/ngi
 	; done \
 	&& cp -a /tmp/build/log-default.json /etc/nginx
 
+# run only on dos build
+RUN --mount=target=/tmp [ -n "${BUILD_OS##*dos*}" ] && exit 0; mkdir -p /root/app_protect_dos /etc/nginx/dos/policies /etc/nginx/dos/logconfs /shared/cores /var/log/adm /var/run/adm && chmod 777 /shared/cores /var/log/adm /var/run/adm /etc/app_protect_dos
+
 RUN --mount=target=/tmp mkdir -p /var/lib/nginx /etc/nginx/secrets /etc/nginx/stream-conf.d \
 	&& setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \
 	&& setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug \

diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go
@@ -70,6 +70,14 @@ var (
 
 	appProtect = flag.Bool("enable-app-protect", false, "Enable support for NGINX App Protect. Requires -nginx-plus.")
 
+	appProtectDos = flag.Bool("enable-app-protect-dos", false, "Enable support for NGINX App Protect dos. Requires -nginx-plus.")
+
+	appProtectDosDebug = flag.Bool("app-protect-dos-debug", false, "Enable debugging for App Protect Dos. Requires -nginx-plus and -enable-app-protect-dos.")
+
+	appProtectDosMaxDaemons = flag.Int("app-protect-dos-max-daemons", 0, "Max number of ADMD instances. Requires -nginx-plus and -enable-app-protect-dos.")
+	appProtectDosMaxWorkers = flag.Int("app-protect-dos-max-workers", 0, "Max number of nginx processes to support. Requires -nginx-plus and -enable-app-protect-dos.")
+	appProtectDosMemory = flag.Int("app-protect-dos-memory", 0, "RAM memory size to consume in MB. Requires -nginx-plus and -enable-app-protect-dos.")
+
 	ingressClass = flag.String("ingress-class", "nginx",
 		`A class of the Ingress controller.
 
@@ -239,6 +247,26 @@ func main() {
 		glog.Fatal("NGINX App Protect support is for NGINX Plus only")
 	}
 
+	if *appProtectDos && !*nginxPlus {
+		glog.Fatal("NGINX App Protect Dos support is for NGINX Plus only")
+	}
+
+	if *appProtectDosDebug && !*appProtectDos && !*nginxPlus {
+		glog.Fatal("NGINX App Protect Dos debug support is for NGINX Plus only and App Protect Dos is enable")
+	}
+
+	if *appProtectDosMaxDaemons != 0 && !*appProtectDos && !*nginxPlus {
+		glog.Fatal("NGINX App Protect Dos max daemons support is for NGINX Plus only and App Protect Dos is enable")
+	}
+
+	if *appProtectDosMaxWorkers != 0 && !*appProtectDos && !*nginxPlus {
+		glog.Fatal("NGINX App Protect Dos max workers support is for NGINX Plus and App Protect Dos is enable")
+	}
+
+	if *appProtectDosMemory != 0 && !*appProtectDos && !*nginxPlus {
+		glog.Fatal("NGINX App Protect Dos memory support is for NGINX Plus and App Protect Dos is enable")
+	}
+
 	if *spireAgentAddress != "" && !*nginxPlus {
 		glog.Fatal("spire-agent-address support is for NGINX Plus only")
 	}
@@ -303,7 +331,7 @@ func main() {
 	}
 
 	var dynClient dynamic.Interface
-	if *appProtect || *ingressLink != "" {
+	if *appProtectDos || *appProtect || *ingressLink != "" {
 		dynClient, err = dynamic.NewForConfig(config)
 		if err != nil {
 			glog.Fatalf("Failed to create dynamic client: %v.", err)
@@ -423,6 +451,13 @@ func main() {
 		nginxManager.AppProtectPluginStart(aPPluginDone)
 	}
 
+	var aPPDosAgentDone chan error
+
+	if *appProtectDos {
+		aPPDosAgentDone = make(chan error, 1)
+		nginxManager.AppProtectDosAgentStart(aPPDosAgentDone, *appProtectDosDebug, *appProtectDosMaxDaemons, *appProtectDosMaxWorkers, *appProtectDosMemory)
+	}
+
 	var sslRejectHandshake bool
 
 	if *defaultServerSecret != "" {
@@ -487,7 +522,7 @@ func main() {
 		if err != nil {
 			glog.Fatalf("Error when getting %v: %v", *nginxConfigMaps, err)
 		}
-		cfgParams = configs.ParseConfigMap(cfm, *nginxPlus, *appProtect)
+		cfgParams = configs.ParseConfigMap(cfm, *nginxPlus, *appProtect, *appProtectDos)
 		if cfgParams.MainServerSSLDHParamFileContent != nil {
 			fileName, err := nginxManager.CreateDHParam(*cfgParams.MainServerSSLDHParamFileContent)
 			if err != nil {
@@ -520,6 +555,7 @@ func main() {
 		EnableSnippets:                 *enableSnippets,
 		NginxServiceMesh:               *spireAgentAddress != "",
 		MainAppProtectLoadModule:       *appProtect,
+		MainAppProtectDosLoadModule:    *appProtectDos,
 		EnableLatencyMetrics:           *enableLatencyMetrics,
 		EnablePreviewPolicies:          *enablePreviewPolicies,
 		SSLRejectHandshake:             sslRejectHandshake,
@@ -616,6 +652,7 @@ func main() {
 		NginxConfigurator:            cnf,
 		DefaultServerSecret:          *defaultServerSecret,
 		AppProtectEnabled:            *appProtect,
+		AppProtectDosEnabled:         *appProtectDos,
 		IsNginxPlus:                  *nginxPlus,
 		IngressClass:                 *ingressClass,
 		ExternalServiceName:          *externalService,
@@ -652,8 +689,8 @@ func main() {
 		}()
 	}
 
-	if *appProtect {
-		go handleTerminationWithAppProtect(lbc, nginxManager, syslogListener, nginxDone, aPAgentDone, aPPluginDone)
+	if *appProtect || *appProtectDos {
+		go handleTerminationWithAppProtect(lbc, nginxManager, syslogListener, nginxDone, aPAgentDone, aPPluginDone, aPPDosAgentDone, *appProtect, *appProtectDos)
 	} else {
 		go handleTermination(lbc, nginxManager, syslogListener, nginxDone)
 	}
@@ -811,7 +848,7 @@ func validateLocation(location string) error {
 	return nil
 }
 
-func handleTerminationWithAppProtect(lbc *k8s.LoadBalancerController, nginxManager nginx.Manager, listener metrics.SyslogListener, nginxDone, agentDone, pluginDone chan error) {
+func handleTerminationWithAppProtect(lbc *k8s.LoadBalancerController, nginxManager nginx.Manager, listener metrics.SyslogListener, nginxDone, agentDone, pluginDone, agentDosDone chan error, appProtectEnabled, appProtectDosEnabled bool) {
 	signalChan := make(chan os.Signal, 1)
 	signal.Notify(signalChan, syscall.SIGTERM)
 
@@ -822,15 +859,23 @@ func handleTerminationWithAppProtect(lbc *k8s.LoadBalancerController, nginxManag
 		glog.Fatalf("AppProtectPlugin command exited unexpectedly with status: %v", err)
 	case err := <-agentDone:
 		glog.Fatalf("AppProtectAgent command exited unexpectedly with status: %v", err)
+	case err := <-agentDosDone:
+		glog.Fatalf("AppProtectDosAgent command exited unexpectedly with status: %v", err)
 	case <-signalChan:
 		glog.Infof("Received SIGTERM, shutting down")
 		lbc.Stop()
 		nginxManager.Quit()
 		<-nginxDone
-		nginxManager.AppProtectPluginQuit()
-		<-pluginDone
-		nginxManager.AppProtectAgentQuit()
-		<-agentDone
+		if appProtectEnabled {
+			nginxManager.AppProtectPluginQuit()
+			<-pluginDone
+			nginxManager.AppProtectAgentQuit()
+			<-agentDone
+		}
+		if appProtectDosEnabled {
+			nginxManager.AppProtectDosAgentQuit()
+			<-agentDosDone
+		}
 		listener.Stop()
 	}
 	glog.Info("Exiting successfully")

diff --git a/deployments/common/crds/appprotectdos.f5.com_apdoslogconfs.yaml b/deployments/common/crds/appprotectdos.f5.com_apdoslogconfs.yaml
@@ -0,0 +1,70 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+  creationTimestamp: null
+  name: apdoslogconfs.appprotectdos.f5.com
+spec:
+  group: appprotectdos.f5.com
+  names:
+    kind: APDosLogConf
+    listKind: APDosLogConfList
+    plural: apdoslogconfs
+    singular: apdoslogconf
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: APDosLogConf is the Schema for the APDosLogConfs API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: APDosLogConfSpec defines the desired state of APDosLogConf
+              properties:
+                content:
+                  properties:
+                    format:
+                      enum:
+                        - splunk
+                        - arcsight
+                        - user-defined
+                      default: splunk
+                      type: string
+                    format_string:
+                      type: string
+                    max_message_size:
+                      pattern: ^([1-9]|[1-5][0-9]|6[0-4])k$
+                      default: 5k
+                      type: string
+                  type: object
+                filter:
+                  properties:
+                    traffic-mitigation-stats:
+                      enum:
+                        - none
+                        - all
+                      default: all
+                      type: string
+                    bad-actors:
+                      pattern: ^(none|all|top ([1-9]|[1-9][0-9]|[1-9][0-9]{2,4}|100000))$
+                      default: top 10
+                      type: string
+                    attack-signatures:
+                      pattern: ^(none|all|top ([1-9]|[1-9][0-9]|[1-9][0-9]{2,4}|100000))$
+                      default: top 10
+                      type: string
+                  type: object
+              type: object
+          type: object
+      served: true
+      storage: true