Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add e2e test suite to detect memory leaks in lua #4236

Merged
merged 1 commit into from
Jun 28, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 6 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ E2E_NODES ?= 10
SLOW_E2E_THRESHOLD ?= 50
K8S_VERSION ?= v1.14.1

E2E_CHECK_LEAKS ?=

ifeq ($(GOHOSTOS),darwin)
SED_I=sed -i ''
endif
Expand Down Expand Up @@ -70,6 +72,9 @@ export GOBUILD_FLAGS
export REPO_INFO
export BUSTED_ARGS
export IMAGE
export E2E_NODES
export E2E_CHECK_LEAKS
export SLOW_E2E_THRESHOLD

# Set default base image dynamically for each arch
BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90
Expand Down Expand Up @@ -174,28 +179,7 @@ lua-test:

.PHONY: e2e-test
e2e-test:
echo "Granting permissions to ingress-nginx e2e service account..."
kubectl create serviceaccount ingress-nginx-e2e || true
kubectl create clusterrolebinding permissive-binding \
--clusterrole=cluster-admin \
--user=admin \
--user=kubelet \
--serviceaccount=default:ingress-nginx-e2e || true

until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
echo "waiting for api token"; \
sleep 3; \
done

kubectl run --rm \
--attach \
--restart=Never \
--generator=run-pod/v1 \
--env="E2E_NODES=$(E2E_NODES)" \
--env="FOCUS=$(FOCUS)" \
--env="SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD)" \
--overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
e2e --image=nginx-ingress-controller:e2e
@build/run-e2e-suite.sh

.PHONY: e2e-test-image
e2e-test-image: e2e-test-binary
Expand Down
80 changes: 80 additions & 0 deletions build/run-e2e-suite.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/bin/bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if ! [ -z "$DEBUG" ]; then
set -x
fi

set -o errexit
set -o nounset
set -o pipefail

RED='\e[35m'
NC='\e[0m'
BGREEN='\e[32m'

declare -a mandatory
mandatory=(
E2E_NODES
SLOW_E2E_THRESHOLD
)

missing=false
for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
echo -e "${RED}Environment variable $var must be set${NC}"
missing=true
fi
done

if [ "$missing" = true ]; then
exit 1
fi

function cleanup {
kubectl delete pod e2e 2>/dev/null || true
}
trap cleanup EXIT

E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-}
FOCUS=${FOCUS:-.*}

export E2E_CHECK_LEAKS
export FOCUS

echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
kubectl create serviceaccount ingress-nginx-e2e || true
kubectl create clusterrolebinding permissive-binding \
--clusterrole=cluster-admin \
--user=admin \
--user=kubelet \
--serviceaccount=default:ingress-nginx-e2e || true

until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \
echo -e "waiting for api token"; \
sleep 3; \
done

kubectl run --rm \
--attach \
--restart=Never \
--generator=run-pod/v1 \
--env="E2E_NODES=${E2E_NODES}" \
--env="FOCUS=${FOCUS}" \
--env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
--env="SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD}" \
--overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
e2e --image=nginx-ingress-controller:e2e
2 changes: 0 additions & 2 deletions rootfs/etc/nginx/template/nginx.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ http {
{{ buildLuaSharedDictionaries $servers $all.Cfg.DisableLuaRestyWAF }}

init_by_lua_block {
require("resty.core")
collectgarbage("collect")

{{ if not $all.Cfg.DisableLuaRestyWAF }}
Expand Down Expand Up @@ -632,7 +631,6 @@ stream {
lua_shared_dict tcp_udp_configuration_data 5M;

init_by_lua_block {
require("resty.core")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

collectgarbage("collect")

-- init modules
Expand Down
58 changes: 36 additions & 22 deletions test/e2e-image/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,36 +16,50 @@

set -e

NC='\e[0m'
BGREEN='\e[32m'

SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-50}
FOCUS=${FOCUS:-.*}
E2E_NODES=${E2E_NODES:-5}
E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""}

if [ ! -f ${HOME}/.kube/config ]; then
kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
kubectl config set-context default --cluster=dev --user=user
kubectl config use-context default
if [ ! -f "${HOME}/.kube/config" ]; then
kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/"
kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
kubectl config set-context default --cluster=dev --user=user
kubectl config use-context default
fi

ginkgo_args=(
"-randomizeSuites"
"-randomizeAllSpecs"
"-flakeAttempts=2"
"-p"
"-trace"
"--noColor=true"
"-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
"-randomizeSuites"
"-randomizeAllSpecs"
"-flakeAttempts=2"
"-p"
"-trace"
"-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
"-r"
)

echo "Running e2e test suite..."
ginkgo "${ginkgo_args[@]}" \
-focus=${FOCUS} \
-skip="\[Serial\]" \
-nodes=${E2E_NODES} \
/e2e.test
echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}"
ginkgo "${ginkgo_args[@]}" \
-focus="${FOCUS}" \
-skip="\[Serial\]|\[MemoryLeak\]" \
-nodes="${E2E_NODES}" \
/e2e.test

echo "Running e2e test suite with tests that require serial execution..."
ginkgo "${ginkgo_args[@]}" \
-focus="\[Serial\]" \
-nodes=1 \
echo -e "${BGREEN}Running e2e test suite with tests that require serial execution...${NC}"
ginkgo "${ginkgo_args[@]}" \
-focus="\[Serial\]" \
-skip="\[MemoryLeak\]" \
-nodes=1 \
/e2e.test

if [[ ${E2E_CHECK_LEAKS} != "" ]]; then
echo -e "${BGREEN}Running e2e test suite with tests that check for memory leaks...${NC}"
ginkgo "${ginkgo_args[@]}" \
-focus="\[MemoryLeak\]" \
-skip="\[Serial\]" \
-nodes=1 \
/e2e.test
fi
1 change: 1 addition & 0 deletions test/e2e/e2e.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
_ "k8s.io/ingress-nginx/test/e2e/dbg"
_ "k8s.io/ingress-nginx/test/e2e/defaultbackend"
_ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown"
_ "k8s.io/ingress-nginx/test/e2e/leaks"
_ "k8s.io/ingress-nginx/test/e2e/loadbalance"
_ "k8s.io/ingress-nginx/test/e2e/lua"
_ "k8s.io/ingress-nginx/test/e2e/servicebackend"
Expand Down
7 changes: 6 additions & 1 deletion test/e2e/framework/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,12 @@ func (f *Framework) AfterEach() {

// IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing.
func IngressNginxDescribe(text string, body func()) bool {
return Describe("[nginx-ingress] "+text, body)
return Describe("[ingress-nginx] "+text, body)
}

// MemoryLeakIt is wrapper function for ginkgo It. Adds "[MemoryLeak]" tag and makes static analysis easier.
func MemoryLeakIt(text string, body interface{}, timeout ...float64) bool {
return It(text+" [MemoryLeak]", body, timeout...)
}

// GetNginxIP returns the number of TCP port where NGINX is running
Expand Down
130 changes: 130 additions & 0 deletions test/e2e/leaks/lua_ssl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package leaks

import (
"crypto/tls"
"fmt"
"net/http"
"strings"
"time"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"github.com/parnurzeal/gorequest"
pool "gopkg.in/go-playground/pool.v3"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("DynamicCertificates", func() {
f := framework.NewDefaultFramework("lua-dynamic-certificates")

BeforeEach(func() {
f.NewEchoDeployment()
})

AfterEach(func() {
})

framework.MemoryLeakIt("should not leak memory from ingress SSL certificates or configuration updates", func() {
hostCount := 1000
iterations := 10

By("Waiting a minute before starting the test")
time.Sleep(1 * time.Minute)

for iteration := 1; iteration <= iterations; iteration++ {
By(fmt.Sprintf("Running iteration %v", iteration))

p := pool.NewLimited(200)

batch := p.Batch()

for index := 1; index <= hostCount; index++ {
host := fmt.Sprintf("hostname-%v", index)
batch.Queue(run(host, f))
}

batch.QueueComplete()
batch.WaitAll()

p.Close()

By("waiting one minute before next iteration")
time.Sleep(1 * time.Minute)
}
})
})

func privisionIngress(hostname string, f *framework.Framework) {
ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostname, "/", hostname, []string{hostname}, f.Namespace, "http-svc", 80, nil))
_, err := framework.CreateIngressTLSSecret(f.KubeClientSet,
ing.Spec.TLS[0].Hosts,
ing.Spec.TLS[0].SecretName,
ing.Namespace)
Expect(err).NotTo(HaveOccurred())

f.WaitForNginxServer(hostname,
func(server string) bool {
return strings.Contains(server, fmt.Sprintf("server_name %v", hostname)) &&
strings.Contains(server, "listen 443")
})
}

func checkIngress(hostname string, f *framework.Framework) {
req := gorequest.New()
resp, _, errs := req.
Get(f.GetURL(framework.HTTPS)).
TLSClientConfig(&tls.Config{ServerName: hostname, InsecureSkipVerify: true}).
Set("Host", hostname).
End()
Expect(errs).Should(BeEmpty())
Expect(resp.StatusCode).Should(Equal(http.StatusOK))

// check the returned secret is not the fake one
cert := resp.TLS.PeerCertificates[0]
Expect(cert.DNSNames[0]).Should(Equal(hostname))
}

func deleteIngress(hostname string, f *framework.Framework) {
err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Delete(hostname, &metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress")
}

func run(host string, f *framework.Framework) pool.WorkFunc {
return func(wu pool.WorkUnit) (interface{}, error) {
if wu.IsCancelled() {
return nil, nil
}

By(fmt.Sprintf("\tcreating ingress for host %v", host))
privisionIngress(host, f)

time.Sleep(100 * time.Millisecond)

By(fmt.Sprintf("\tchecking ingress for host %v", host))
checkIngress(host, f)

By(fmt.Sprintf("\tdestroying ingress for host %v", host))
deleteIngress(host, f)

return true, nil
}
}