From dbc3f007aa05ad0b5f59a8403c11cd1a76aaeeb3 Mon Sep 17 00:00:00 2001 From: tkrop Date: Wed, 3 Apr 2019 16:38:45 +0200 Subject: [PATCH 01/37] feat: upgrade skipper to v0.10.200 (#1965) Signed-off-by: tkrop --- cluster/manifests/skipper/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 673049bc7a..30e73d5679 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: skipper-ingress - version: v0.10.195 + version: v0.10.200 component: ingress spec: strategy: @@ -18,7 +18,7 @@ spec: metadata: labels: application: skipper-ingress - version: v0.10.195 + version: v0.10.200 component: ingress annotations: kubernetes-log-watcher/scalyr-parser: | @@ -42,7 +42,7 @@ spec: hostNetwork: true containers: - name: skipper-ingress - image: registry.opensource.zalan.do/pathfinder/skipper:v0.10.195 + image: registry.opensource.zalan.do/pathfinder/skipper:v0.10.200 ports: - name: ingress-port containerPort: 9999 From a55234b2e79d9d634340f32480ddab26b0a138ba Mon Sep 17 00:00:00 2001 From: Maxim Tschumak Date: Thu, 4 Apr 2019 11:23:11 +0200 Subject: [PATCH 02/37] feat: configure Skipper to use default filters (#1965) Signed-off-by: Maxim Tschumak --- cluster/manifests/skipper/deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 30e73d5679..8c9cbb40a8 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -80,6 +80,7 @@ spec: - "-api-usage-monitoring-realm-keys=https://identity.zalando.com/realm" - "-api-usage-monitoring-client-keys=https://identity.zalando.com/managed-id,sub" - "-api-usage-monitoring-default-client-tracking-pattern=services[.].*" + - "-default-filters-dir=/etc/config/default-filters" {{ end }} - "-max-audit-body=0" {{ if eq .ConfigItems.skipper_clusterratelimit "true"}} From 432eddceab2a928e71695cff7f19e9e1f6fecf78 Mon Sep 17 00:00:00 2001 From: Henning Jacobs Date: Thu, 4 Apr 2019 21:41:56 +0200 Subject: [PATCH 03/37] kube-downscaler v0.12 Signed-off-by: Henning Jacobs --- cluster/manifests/kube-downscaler/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/kube-downscaler/deployment.yaml b/cluster/manifests/kube-downscaler/deployment.yaml index 97b3ceb7e2..9f54006ac0 100644 --- a/cluster/manifests/kube-downscaler/deployment.yaml +++ b/cluster/manifests/kube-downscaler/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-downscaler - version: v0.6 + version: v0.12 spec: replicas: 1 selector: @@ -15,7 +15,7 @@ spec: metadata: labels: application: kube-downscaler - version: v0.7 + version: v0.12 spec: dnsConfig: options: @@ -26,7 +26,7 @@ spec: containers: - name: downscaler # see https://github.com/hjacobs/kube-downscaler/releases - image: registry.opensource.zalan.do/teapot/kube-downscaler:0.7 + image: registry.opensource.zalan.do/teapot/kube-downscaler:0.12 args: - --interval=30 - --exclude-namespaces=kube-system,visibility From b121c80eb6f3b01b94f3abe9003db40c9b109172 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Thu, 4 Apr 2019 22:32:42 +0200 Subject: [PATCH 04/37] Only run e2e on PR Signed-off-by: Mikkel Oscar Lyderik Larsen --- delivery.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/delivery.yaml b/delivery.yaml index c1e5c7fd7b..f4b7d09a7d 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -2,6 +2,8 @@ version: "2017-09-20" allow_concurrent_steps: true pipeline: - id: build + when: + event: pull_request vm: large # speed up building kubernetes/kubernetes overlay: ci/golang cache: @@ -15,6 +17,8 @@ pipeline: VERSION="$CDP_BUILD_VERSION" make -C test/e2e build.push - id: e2e-tests + when: + event: pull_request depends_on: [build] type: process desc: "Kubernetes e2e tests" From 16208e7558e3c51aa736206f2f0e0f7729fe746e Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 5 Apr 2019 08:19:15 +0200 Subject: [PATCH 05/37] Enable Egress e2e tests Signed-off-by: Mikkel Oscar Lyderik Larsen --- test/e2e/README.md | 2 +- test/e2e/go.mod | 5 ++++- test/e2e/go.sum | 10 ++++++++-- test/e2e/run_e2e.sh | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 4398f8a9e3..178beb2d11 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -27,7 +27,7 @@ examples of how to write the tests or checkout the files already defined e.g. KUBECONFIG=~/.kube/config HOSTED_ZONE=example.org \ ginkgo -nodes=25 -flakeAttempts=2 \ -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*CockroachDB|\[Zalando\])" \ - -skip="(\[Serial\]|\[Egress\])" \ + -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false ``` diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 63241a175c..8c60d4153a 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -81,6 +81,7 @@ require ( github.com/heketi/rest v0.0.0-20180404230133-aa6a65207413 // indirect github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 // indirect github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64 // indirect + github.com/hpcloud/tail v1.0.0 // indirect github.com/imdario/mergo v0.3.5 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20151117175822-3433f3ea46d9 // indirect @@ -106,7 +107,7 @@ require ( github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/natefinch/lumberjack v2.0.0+incompatible // indirect - github.com/onsi/ginkgo v1.4.0 + github.com/onsi/ginkgo v1.8.0 github.com/onsi/gomega v1.2.0 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect @@ -158,11 +159,13 @@ require ( google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect google.golang.org/grpc v1.15.0 // indirect gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/gcfg.v1 v1.2.0 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect gopkg.in/inf.v0 v0.9.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7 // indirect gopkg.in/square/go-jose.v2 v2.2.0 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect gotest.tools v2.2.0+incompatible // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 61768faebe..5502c10b89 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -194,6 +194,8 @@ github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 h1:oJ/NLadJn5HoxvonA6 github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64 h1:dk3GEa55HcRVIyCeNQmwwwH3kIXnqJPNseKOkDD+7uQ= github.com/heketi/utils v0.0.0-20170317161834-435bc5bdfa64/go.mod h1:RYlF4ghFZPPmk2TC5REt5OFwvfb6lzxFWrTWB+qs28s= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -249,8 +251,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/onsi/ginkgo v1.4.0 h1:n60/4GZK0Sr9O2iuGKq876Aoa0ER2ydgpMOBwzJ8e2c= -github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.2.0 h1:tQjc4uvqBp0z424R9V/S2L18penoUiwZftoY0t48IZ4= github.com/onsi/gomega v1.2.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= @@ -376,6 +378,8 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNat gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0= gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= @@ -386,6 +390,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7 h1:986b60BAz gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.2.0 h1:0kdiskBe/uJirf0T5GGmZlS8bWRYUszavQpx91WycKs= gopkg.in/square/go-jose.v2 v2.2.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index dcb989f44f..ca07ef2aef 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -105,7 +105,7 @@ export KUBECONFIG="$KUBECONFIG" # * custom 'zalando' tests ginkgo -nodes=25 -flakeAttempts=2 \ -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*CockroachDB|\[Zalando\])" \ - -skip="(\[Serial\]|\[Egress\])" \ + -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false # delete cluster From c9ca4bf203d826320ef6ce542abbd1066cb018fd Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Fri, 5 Apr 2019 13:24:56 +0200 Subject: [PATCH 06/37] Update CA Signed-off-by: Alexey Ermakov --- cluster/manifests/kube-cluster-autoscaler/daemonset.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml index a17b6844e5..1df1d8fb56 100644 --- a/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml +++ b/cluster/manifests/kube-cluster-autoscaler/daemonset.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-cluster-autoscaler - version: v1.12.2-internal49 + version: v1.12.2-internal63 spec: selector: matchLabels: @@ -16,7 +16,7 @@ spec: metadata: labels: application: kube-cluster-autoscaler - version: v1.12.2-internal49 + version: v1.12.2-internal63 annotations: iam.amazonaws.com/role: "{{ .LocalID }}-app-autoscaler" config/pool-sizes: "{{range .NodePools}}{{.Name}}-{{.MinSize}}-{{.MaxSize}} {{end}}" @@ -33,7 +33,7 @@ spec: effect: NoSchedule containers: - name: cluster-autoscaler - image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:v1.12.2-internal49 + image: registry.opensource.zalan.do/teapot/kube-cluster-autoscaler:v1.12.2-internal63 command: - ./cluster-autoscaler - --v=4 From cd3e5589cb8337feae6e5c3629c81f3f106a782d Mon Sep 17 00:00:00 2001 From: Arjun Naik Date: Wed, 3 Apr 2019 11:37:23 +0200 Subject: [PATCH 07/37] Updated the version of kube-metrics-adapter Signed-off-by: Arjun Naik --- cluster/manifests/kube-metrics-adapter/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/manifests/kube-metrics-adapter/deployment.yaml b/cluster/manifests/kube-metrics-adapter/deployment.yaml index 2740965b82..a889328ede 100644 --- a/cluster/manifests/kube-metrics-adapter/deployment.yaml +++ b/cluster/manifests/kube-metrics-adapter/deployment.yaml @@ -25,7 +25,7 @@ spec: serviceAccountName: custom-metrics-apiserver containers: - name: kube-metrics-adapter - image: registry.opensource.zalan.do/teapot/kube-metrics-adapter:master-20 + image: registry.opensource.zalan.do/teapot/kube-metrics-adapter:master-25 args: - --prometheus-server=http://prometheus.kube-system.svc.cluster.local - --skipper-ingress-metrics From 5e8296c7d7081fdf1e518b66c1fe1176939707ed Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Fri, 5 Apr 2019 15:52:13 +0200 Subject: [PATCH 08/37] Add a config item for compliance checker integration Signed-off-by: Alexey Ermakov --- cluster/config-defaults.yaml | 1 + cluster/node-pools/master-default/userdata.clc.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index b26e332027..3821156dd1 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -62,6 +62,7 @@ image_policy: "trusted" {{else}} image_policy: "dev" {{end}} +compliance_checker_enabled: "false" # Egress configuration nat_cidr_blocks: "172.31.64.0/28,172.31.64.16/28,172.31.64.32/28" diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index 55d5b8497b..c1eb35df61 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -501,7 +501,7 @@ storage: value: https://identity.zalando.com/.well-known/openid-configuration - name: ENABLE_INTROSPECTION value: "true" - - image: registry.opensource.zalan.do/teapot/image-policy-webhook:v0.4.1 + - image: registry.opensource.zalan.do/teapot/image-policy-webhook:{{if eq .Cluster.ConfigItems.compliance_checker_enabled "true"}}master-44{{else}}v0.4.1{{end}} name: image-policy-webhook args: - --policy={{ .Cluster.ConfigItems.image_policy }} From 75e3c6e60b04b7638cd9d3aca4eefa2df58d1cc7 Mon Sep 17 00:00:00 2001 From: Arjun Naik Date: Fri, 5 Apr 2019 16:45:48 +0200 Subject: [PATCH 09/37] Increased memory to prevent OOMKills Signed-off-by: Arjun Naik --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index c1e5c7fd7b..5910c25993 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -45,7 +45,7 @@ pipeline: resources: &resources limits: cpu: 500m - memory: 2Gi + memory: 4Gi requests: cpu: 500m - memory: 2Gi + memory: 4Gi From 264ebcc638bb47a940a522deec3303255de6e4d3 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 5 Apr 2019 17:17:20 +0200 Subject: [PATCH 10/37] Use compliant image Signed-off-by: Mikkel Oscar Lyderik Larsen --- test/e2e/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index d5e8145780..cb444ee262 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -132,7 +132,7 @@ func createPingPod(nameprefix, namespace string) *v1.Pod { Containers: []v1.Container{ { Name: "check-change-myip", - Image: "registry.opensource.zalan.do/teapot/check-change-myip:v0.0.1", + Image: "registry.opensource.zalan.do/teapot/check-change-myip:master-2", }, }, }, From 0842475dcf9f7999b4cbc68924e7d4e6a38ab686 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Sun, 7 Apr 2019 18:17:15 +0200 Subject: [PATCH 11/37] Update prometheus to v2.8.1 https://github.com/prometheus/prometheus/releases Some of these changes seem to be interesting enough to update [ENHANCEMENT] Query performance improvements. prometheus/tsdb#531 [BUGFIX] Scrape: catch errors when creating HTTP clients #5182. Adds new metrics: prometheus_target_scrape_pools_* deprecating the flag storage.tsdb.retention -> use storage.tsdb.retention.time [FEATURE] Add subqueries to PromQL. [ENHANCEMENT] Kubernetes SD: Add service external IP and external name to the discovery metadata. #4940 [ENHANCEMENT] Add metric for number of rule groups loaded. #5090 BUGFIX] Make sure the retention period does not overflow. #5112 [BUGFIX] Make sure the blocks do not get very large. #5112 [BUGFIX] Do not generate blocks with no samples. prometheus/tsdb#374 [BUGFIX] Reintroduce metric for WAL corruptions. prometheus/tsdb#473 Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/prometheus/statefulset.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/prometheus/statefulset.yaml b/cluster/manifests/prometheus/statefulset.yaml index af02723f01..b7daedbb91 100644 --- a/cluster/manifests/prometheus/statefulset.yaml +++ b/cluster/manifests/prometheus/statefulset.yaml @@ -4,7 +4,7 @@ metadata: annotations: labels: application: prometheus - version: v2.6.0 + version: v2.8.1 name: prometheus namespace: kube-system spec: @@ -17,7 +17,7 @@ spec: metadata: labels: application: prometheus - version: v2.6.0 + version: v2.8.1 annotations: config/hash: {{"configmap.yaml" | manifestHash}} spec: @@ -28,7 +28,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: prometheus - image: registry.opensource.zalan.do/teapot/prometheus:v2.6.0 + image: registry.opensource.zalan.do/teapot/prometheus:v2.8.1 args: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus/" From 4767c6c43f8ba1018efac792f233ad6695a7bfae Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Mon, 8 Apr 2019 11:36:19 +0200 Subject: [PATCH 12/37] Enable DS scheduling for e2e Signed-off-by: Alexey Ermakov --- cluster/config-defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 3821156dd1..065f5778d9 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -191,4 +191,8 @@ coreos_image: "ami-012abdf0d2781f0a5" # Container Linux 2023.5.0 (HVM, eu-centra enable_ingress_template_controller: "false" # Temporary feature toggle for the new daemonset scheduler +{{if eq .Environment "e2e"}} +experimental_schedule_daemonset_pods: "true" +{{else}} experimental_schedule_daemonset_pods: "false" +{{end}} From 8c5ddf4bf4987e126be1de277e04d2b80f759557 Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Tue, 9 Apr 2019 11:17:19 +0200 Subject: [PATCH 13/37] Increase nvidia-driver-installer yet again Signed-off-by: Alexey Ermakov --- cluster/manifests/nvidia/nvidia-driver-installer.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/manifests/nvidia/nvidia-driver-installer.yaml b/cluster/manifests/nvidia/nvidia-driver-installer.yaml index e60a75ae1f..dc7fd03e74 100644 --- a/cluster/manifests/nvidia/nvidia-driver-installer.yaml +++ b/cluster/manifests/nvidia/nvidia-driver-installer.yaml @@ -53,10 +53,10 @@ spec: resources: limits: cpu: 150m - memory: 512Mi + memory: 768Mi requests: cpu: 150m - memory: 512Mi + memory: 768Mi securityContext: privileged: true env: @@ -82,7 +82,7 @@ spec: resources: limits: cpu: 150m - memory: 512Mi + memory: 768Mi requests: cpu: 150m - memory: 512Mi + memory: 768Mi From 1ecf3f22446ff247f613138dcdcf45af21080f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 9 Apr 2019 11:48:36 +0200 Subject: [PATCH 14/37] fix memory consumption for east west routes https://github.com/zalando/skipper/releases/tag/v0.10.203 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/manifests/skipper/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 8c9cbb40a8..235347c255 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: skipper-ingress - version: v0.10.200 + version: v0.10.203 component: ingress spec: strategy: @@ -18,7 +18,7 @@ spec: metadata: labels: application: skipper-ingress - version: v0.10.200 + version: v0.10.203 component: ingress annotations: kubernetes-log-watcher/scalyr-parser: | @@ -42,7 +42,7 @@ spec: hostNetwork: true containers: - name: skipper-ingress - image: registry.opensource.zalan.do/pathfinder/skipper:v0.10.200 + image: registry.opensource.zalan.do/pathfinder/skipper:v0.10.203 ports: - name: ingress-port containerPort: 9999 From 8e78e4b553110ae9d9a1bc363ea7b6bb39f33bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 9 Apr 2019 12:07:34 +0200 Subject: [PATCH 15/37] fix external-dns settings to the values measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/manifests/external-dns/deployment.yaml | 1 + cluster/manifests/external-dns/vpa.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster/manifests/external-dns/deployment.yaml b/cluster/manifests/external-dns/deployment.yaml index 837a06bc9e..922e4e9e20 100644 --- a/cluster/manifests/external-dns/deployment.yaml +++ b/cluster/manifests/external-dns/deployment.yaml @@ -35,6 +35,7 @@ spec: - --provider=aws - --registry=txt - --txt-owner-id={{ .Region }}:{{ .LocalID }} + - --aws-batch-change-size=350 resources: limits: cpu: 50m diff --git a/cluster/manifests/external-dns/vpa.yaml b/cluster/manifests/external-dns/vpa.yaml index 6b3d92def1..7a58fee50d 100644 --- a/cluster/manifests/external-dns/vpa.yaml +++ b/cluster/manifests/external-dns/vpa.yaml @@ -16,5 +16,5 @@ spec: containerPolicies: - containerName: external-dns maxAllowed: - cpu: 50m - memory: 100Mi + cpu: 500m + memory: 2Gi From 1d177d536703a00cdc440d958c2e4b40cef3f09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Tue, 9 Apr 2019 12:10:50 +0200 Subject: [PATCH 16/37] add vpa for kube-ingress-alb-controller MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/manifests/ingress-controller/vpa.yaml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 cluster/manifests/ingress-controller/vpa.yaml diff --git a/cluster/manifests/ingress-controller/vpa.yaml b/cluster/manifests/ingress-controller/vpa.yaml new file mode 100644 index 0000000000..3ca4a10046 --- /dev/null +++ b/cluster/manifests/ingress-controller/vpa.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling.k8s.io/v1beta1 +kind: VerticalPodAutoscaler +metadata: + name: kube-ingress-aws-controller + namespace: kube-system + labels: + application: kube-ingress-aws-controller +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: kube-ingress-aws-controller + updatePolicy: + updateMode: Auto + resourcePolicy: + containerPolicies: + - containerName: kube-ingress-aws-controller + maxAllowed: + cpu: 250m + memory: 1Gi From 560d78e2f891d1873230f79837382850fa91a35a Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Fri, 5 Apr 2019 15:52:13 +0200 Subject: [PATCH 17/37] Add a config item for compliance checker integration Signed-off-by: Alexey Ermakov --- cluster/config-defaults.yaml | 1 + cluster/node-pools/master-default/userdata.clc.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index b26e332027..3821156dd1 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -62,6 +62,7 @@ image_policy: "trusted" {{else}} image_policy: "dev" {{end}} +compliance_checker_enabled: "false" # Egress configuration nat_cidr_blocks: "172.31.64.0/28,172.31.64.16/28,172.31.64.32/28" diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index 55d5b8497b..c1eb35df61 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -501,7 +501,7 @@ storage: value: https://identity.zalando.com/.well-known/openid-configuration - name: ENABLE_INTROSPECTION value: "true" - - image: registry.opensource.zalan.do/teapot/image-policy-webhook:v0.4.1 + - image: registry.opensource.zalan.do/teapot/image-policy-webhook:{{if eq .Cluster.ConfigItems.compliance_checker_enabled "true"}}master-44{{else}}v0.4.1{{end}} name: image-policy-webhook args: - --policy={{ .Cluster.ConfigItems.image_policy }} From 00d989a105aebe964d0dbc1d53b9503751dcefc8 Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Tue, 9 Apr 2019 16:54:59 +0200 Subject: [PATCH 18/37] e2e: increase memory Signed-off-by: Alexey Ermakov --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index f7d8b3c82b..fc9a9a0051 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -49,7 +49,7 @@ pipeline: resources: &resources limits: cpu: 500m - memory: 4Gi + memory: 5Gi requests: cpu: 500m - memory: 4Gi + memory: 5Gi From 6d68f367fc36ce0e9748f5ab5670c1c49c1ebd02 Mon Sep 17 00:00:00 2001 From: Arjun Naik Date: Wed, 10 Apr 2019 11:24:48 +0200 Subject: [PATCH 19/37] Updated the VPAs to v1beta2 Signed-off-by: Arjun Naik --- cluster/manifests/external-dns/vpa.yaml | 2 +- cluster/manifests/heapster/vpa.yaml | 2 +- cluster/manifests/ingress-controller/vpa.yaml | 2 +- cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml | 2 +- cluster/manifests/metrics-server/metrics-server-vpa.yaml | 2 +- cluster/manifests/prometheus/prometheus-vpa.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster/manifests/external-dns/vpa.yaml b/cluster/manifests/external-dns/vpa.yaml index 7a58fee50d..47eb490b68 100644 --- a/cluster/manifests/external-dns/vpa.yaml +++ b/cluster/manifests/external-dns/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: external-dns diff --git a/cluster/manifests/heapster/vpa.yaml b/cluster/manifests/heapster/vpa.yaml index d4d913ab95..dd3005d05b 100644 --- a/cluster/manifests/heapster/vpa.yaml +++ b/cluster/manifests/heapster/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: heapster diff --git a/cluster/manifests/ingress-controller/vpa.yaml b/cluster/manifests/ingress-controller/vpa.yaml index 3ca4a10046..660616e2cc 100644 --- a/cluster/manifests/ingress-controller/vpa.yaml +++ b/cluster/manifests/ingress-controller/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: kube-ingress-aws-controller diff --git a/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml b/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml index 4dd7be02b2..c4138d027a 100644 --- a/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml +++ b/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: kubernetes-lifecycle-metrics-vpa diff --git a/cluster/manifests/metrics-server/metrics-server-vpa.yaml b/cluster/manifests/metrics-server/metrics-server-vpa.yaml index a1dc9f62f3..67df6b97a9 100644 --- a/cluster/manifests/metrics-server/metrics-server-vpa.yaml +++ b/cluster/manifests/metrics-server/metrics-server-vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: metrics-server-vpa diff --git a/cluster/manifests/prometheus/prometheus-vpa.yaml b/cluster/manifests/prometheus/prometheus-vpa.yaml index 0e4e5a70ae..c8e91a7c5c 100644 --- a/cluster/manifests/prometheus/prometheus-vpa.yaml +++ b/cluster/manifests/prometheus/prometheus-vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: prometheus-vpa From 0bd16e5cbbd668dec6f50ed8e6db3cbdf7a59534 Mon Sep 17 00:00:00 2001 From: Arjun Naik Date: Wed, 10 Apr 2019 11:24:48 +0200 Subject: [PATCH 20/37] Updated the VPAs to v1beta2 Signed-off-by: Arjun Naik --- cluster/manifests/external-dns/vpa.yaml | 2 +- cluster/manifests/heapster/vpa.yaml | 2 +- cluster/manifests/ingress-controller/vpa.yaml | 2 +- cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml | 2 +- cluster/manifests/metrics-server/metrics-server-vpa.yaml | 2 +- cluster/manifests/prometheus/prometheus-vpa.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster/manifests/external-dns/vpa.yaml b/cluster/manifests/external-dns/vpa.yaml index 7a58fee50d..47eb490b68 100644 --- a/cluster/manifests/external-dns/vpa.yaml +++ b/cluster/manifests/external-dns/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: external-dns diff --git a/cluster/manifests/heapster/vpa.yaml b/cluster/manifests/heapster/vpa.yaml index d4d913ab95..dd3005d05b 100644 --- a/cluster/manifests/heapster/vpa.yaml +++ b/cluster/manifests/heapster/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: heapster diff --git a/cluster/manifests/ingress-controller/vpa.yaml b/cluster/manifests/ingress-controller/vpa.yaml index 3ca4a10046..660616e2cc 100644 --- a/cluster/manifests/ingress-controller/vpa.yaml +++ b/cluster/manifests/ingress-controller/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: kube-ingress-aws-controller diff --git a/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml b/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml index 4dd7be02b2..c4138d027a 100644 --- a/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml +++ b/cluster/manifests/kubernetes-lifecycle-metrics/vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: kubernetes-lifecycle-metrics-vpa diff --git a/cluster/manifests/metrics-server/metrics-server-vpa.yaml b/cluster/manifests/metrics-server/metrics-server-vpa.yaml index a1dc9f62f3..67df6b97a9 100644 --- a/cluster/manifests/metrics-server/metrics-server-vpa.yaml +++ b/cluster/manifests/metrics-server/metrics-server-vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: metrics-server-vpa diff --git a/cluster/manifests/prometheus/prometheus-vpa.yaml b/cluster/manifests/prometheus/prometheus-vpa.yaml index 0e4e5a70ae..c8e91a7c5c 100644 --- a/cluster/manifests/prometheus/prometheus-vpa.yaml +++ b/cluster/manifests/prometheus/prometheus-vpa.yaml @@ -1,4 +1,4 @@ -apiVersion: autoscaling.k8s.io/v1beta1 +apiVersion: autoscaling.k8s.io/v1beta2 kind: VerticalPodAutoscaler metadata: name: prometheus-vpa From 4397e7d37bae8bed2664083c9c1d97b070b820a4 Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Wed, 10 Apr 2019 11:54:51 +0200 Subject: [PATCH 21/37] Enable compliance checker by default Signed-off-by: Alexey Ermakov --- cluster/config-defaults.yaml | 2 +- test/e2e/apiserver.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index 065f5778d9..c770f1bb93 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -62,7 +62,7 @@ image_policy: "trusted" {{else}} image_policy: "dev" {{end}} -compliance_checker_enabled: "false" +compliance_checker_enabled: "true" # Egress configuration nat_cidr_blocks: "172.31.64.0/28,172.31.64.16/28,172.31.64.32/28" diff --git a/test/e2e/apiserver.go b/test/e2e/apiserver.go index 1711fa667a..0ccb2e0fbb 100644 --- a/test/e2e/apiserver.go +++ b/test/e2e/apiserver.go @@ -69,7 +69,7 @@ var _ = framework.KubeDescribe("API Server webhook tests", func() { }) It("Should not allow deployment using not trusted image [Image-Webhook] [Non-Compliant] [Zalando]", func() { - tag := "bc1a6fe-nottrusted" + tag := "bc1a6fe-nottrusted2" nameprefix := "image-policy-webhook-test-non-compliant" podname := fmt.Sprintf("image-webhook-policy-test-pod-%s", tag) From 5bdc4307d27bd5b75d276ff882eb84d7c1198ddb Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Wed, 10 Apr 2019 21:03:22 +0200 Subject: [PATCH 22/37] Update Egress controller to v0.1.6 Signed-off-by: Mikkel Oscar Lyderik Larsen --- .../manifests/kube-static-egress-controller/deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster/manifests/kube-static-egress-controller/deployment.yaml b/cluster/manifests/kube-static-egress-controller/deployment.yaml index eaa1d6cefe..5d1c2c5386 100644 --- a/cluster/manifests/kube-static-egress-controller/deployment.yaml +++ b/cluster/manifests/kube-static-egress-controller/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: kube-static-egress-controller - version: v0.1.5 + version: v0.1.6 spec: replicas: 1 selector: @@ -15,7 +15,7 @@ spec: metadata: labels: application: kube-static-egress-controller - version: v0.1.5 + version: v0.1.6 annotations: iam.amazonaws.com/role: "{{ .LocalID }}-static-egress-controller" spec: @@ -26,7 +26,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: controller - image: registry.opensource.zalan.do/teapot/kube-static-egress-controller:v0.1.5 + image: registry.opensource.zalan.do/teapot/kube-static-egress-controller:v0.1.6 args: - "--log-level=debug" - "--provider=aws" From afb4dcee2ac27f25ef663c3a5c14a411b98d342c Mon Sep 17 00:00:00 2001 From: Christian Berg Date: Thu, 11 Apr 2019 11:22:49 +0200 Subject: [PATCH 23/37] Fluentd config: turn off S3 bucket checks Turn off the feature to create the S3 bucket and to check for S3 bucket existance at startup. The logging-agent doesn't have permission to create S3 buckets, and the check causes the Fluentd worker to fail and restart if the bucket is not present. Constant restarting leads to high CPU consumption. Instead we choos to keep the worker running and try to ship logs to S3. This will fail if the bucket does not exist, but the buffer size is limited and we can alert on buffer size and errors and the CPU consumption stays low. Signed-off-by: Christian Berg --- cluster/manifests/logging-agent/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster/manifests/logging-agent/config.yaml b/cluster/manifests/logging-agent/config.yaml index 9e3f37790f..b1fc14be81 100644 --- a/cluster/manifests/logging-agent/config.yaml +++ b/cluster/manifests/logging-agent/config.yaml @@ -69,6 +69,8 @@ data: @type s3 s3_bucket {{ index .ConfigItems "logging_s3_bucket" }} s3_region eu-central-1 + auto_create_bucket false + check_bucket false retries 10 From d95d7683fc8bc50b2039b2f924cbf97ade24d5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 11 Apr 2019 12:00:20 +0200 Subject: [PATCH 24/37] first simple ingress test that checks that content provided by endpoint is served correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/Makefile | 3 ++ test/e2e/README.md | 2 +- test/e2e/go.mod | 10 ++--- test/e2e/go.sum | 24 +++++++++++ test/e2e/ingress.go | 100 +++++++++++++++++++++++++++++++++++++++++++- test/e2e/util.go | 75 ++++++++++++++++++++++++++++++++- 6 files changed, 206 insertions(+), 8 deletions(-) diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 471c8523d5..cc4972f09b 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -18,6 +18,9 @@ $(MOD_PATH): fix-go-modules: $(MOD_PATH) MOD_PATH=$(MOD_PATH) ./setup_modules.sh +deps: fix-go-modules + GO111MODULE=on go get github.com/onsi/ginkgo/ginkgo@v1.8.0 + e2e.test: fix-go-modules GO111MODULE=on go test -v -c -o e2e.test diff --git a/test/e2e/README.md b/test/e2e/README.md index 178beb2d11..c40d2184f2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -12,7 +12,7 @@ examples of how to write the tests or checkout the files already defined e.g. 1. First you need [ginkgo] which is used to orchestrate the tests: ```bash - GO111MODULE=on go get github.com/onsi/ginkgo/ginkgo@v1.8.0 + make deps ``` 2. Build the e2e test binary: diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 8c60d4153a..26ced78207 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -59,12 +59,12 @@ require ( github.com/go-openapi/validate v0.17.2 // indirect github.com/go-ozzo/ozzo-validation v3.5.0+incompatible // indirect github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55 // indirect - github.com/gogo/protobuf v0.0.0-20170330071051-c0656edd0d9e // indirect + github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect github.com/google/cadvisor v0.31.0 // indirect github.com/google/go-cmp v0.2.0 // indirect - github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 // indirect + github.com/google/gofuzz v1.0.0 // indirect github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9 // indirect github.com/gophercloud/gophercloud v0.0.0-20180330165814-781450b3c4fc // indirect github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect @@ -137,7 +137,7 @@ require ( github.com/spf13/cast v0.0.0-20160730092037-e31f36ffc91a // indirect github.com/spf13/cobra v0.0.0-20160722081547-f62e98d28ab7 // indirect github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80 // indirect - github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff // indirect + github.com/spf13/pflag v1.0.3 // indirect github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8 // indirect github.com/storageos/go-api v0.0.0-20180126153955-3a4032328d99 // indirect github.com/stretchr/objx v0.1.1 // indirect @@ -162,7 +162,7 @@ require ( gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/gcfg.v1 v1.2.0 // indirect gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect - gopkg.in/inf.v0 v0.9.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7 // indirect gopkg.in/square/go-jose.v2 v2.2.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect @@ -171,7 +171,7 @@ require ( gotest.tools v2.2.0+incompatible // indirect k8s.io/api v0.0.0-20170719033850-4d5cc6efc5e8 k8s.io/apiextensions-apiserver v0.0.0-20170616221715-abd4538a1176 // indirect - k8s.io/apimachinery v0.0.0-20170616220319-2c6e1537d30b + k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b k8s.io/apiserver v0.0.0-20170616221227-b1c85a6c288c // indirect k8s.io/cli-runtime v0.0.0-20190301175653-0c2382d2cffa // indirect k8s.io/client-go v0.0.0-20170617220358-c1b6a1c0a439 diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 5502c10b89..91882bc510 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -109,6 +109,7 @@ github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96 h1:5e8GDOdG6jKeeqN github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fsnotify/fsnotify v0.0.0-20160816051541-f12c6236fe7b h1:lHoxUxMozh/yCASOoFep9dPMva62ztmxKK2VB8//Aoo= github.com/fsnotify/fsnotify v0.0.0-20160816051541-f12c6236fe7b/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb h1:D4uzjWwKYQ5XnAvUbuvHW93esHg7F8N/OYeBBcJoTr0= @@ -143,6 +144,11 @@ github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55 h1:oIgNYSrSUbNH5DJh6DM github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v0.0.0-20170330071051-c0656edd0d9e h1:ago6fNuQ6IhszPsXkeU7qRCyfsIX7L67WDybsAPkLl8= github.com/gogo/protobuf v0.0.0-20170330071051-c0656edd0d9e/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20141105023935-44145f04b68c/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= @@ -160,6 +166,10 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367 h1:ScAXWS+TR6MZKex+7Z8rneuSJH+FSDqd6ocQyl+ZHo4= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9 h1:Dk8DO58y0wXeNG49dRqzf4HrZAK9lFfTgS/FNtB6NlM= @@ -204,6 +214,7 @@ github.com/jmespath/go-jmespath v0.0.0-20151117175822-3433f3ea46d9 h1:1SlajWtS+u github.com/jmespath/go-jmespath v0.0.0-20151117175822-3433f3ea46d9/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jteeuwen/go-bindata v3.0.8-0.20180305030458-6025e8de665b+incompatible h1:eX6cWzw+KSwhN430wwbdWPgqnlbnK5ux76/q5ko+Qu8= @@ -212,6 +223,7 @@ github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpR github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro= github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169 h1:YUrU1/jxRqnt0PSrKj1Uj/wEjk/fjnE80QFfi2Zlj7Q= github.com/kr/fs v0.0.0-20131111012553-2788f0dbd169/go.mod h1:glhvuHOU9Hy7/8PwwdtnarXqLagOX0b/TbZx2zLMqEg= @@ -317,6 +329,10 @@ github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80 h1:evyGXhH github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff h1:VARhShG49tiji6mdRNp7JTNDtJ0FhuprF93GBQ37xGU= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8 h1:hh6dlnX5EheZdLFxYsRZt7d2z6lOnP8qVEksE/7UrhM= github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= github.com/storageos/go-api v0.0.0-20180126153955-3a4032328d99 h1:FRurKqd9bRx8aOnMBWRtfqvCYYulMAhBqLTCdH7+8Mw= @@ -353,6 +369,8 @@ golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58 h1:otZG8yDCO4LVps5+9bxOeNiCvgmOyt96J3roHTYs7oE= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 h1:bfLnR+k0tq5Lqt6dflRLcZiz6UaXCMt3vhYJ1l4FQ80= +golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= @@ -362,9 +380,13 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUk golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/api v0.0.0-20181129220737-af4fc4062c26 h1:RPZDMF+4oXwRE3YH8oxQXeORJvVzSLPusZrxbwgpVVg= google.golang.org/api v0.0.0-20181129220737-af4fc4062c26/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= @@ -386,6 +408,8 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNj gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7 h1:986b60BAz5vO2Vaf48yQaq+wb2bU4JsXxKu1+itW6x8= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.2.0 h1:0kdiskBe/uJirf0T5GGmZlS8bWRYUszavQpx91WycKs= diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index de5dd006b1..98dcfd9499 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -14,7 +14,11 @@ limitations under the License. package e2e import ( + "bytes" "fmt" + "io" + "log" + "net/http" "time" . "github.com/onsi/ginkgo" @@ -101,8 +105,102 @@ var _ = framework.KubeDescribe("Ingress ALB creation", func() { Expect(err).NotTo(HaveOccurred()) // DNS ready - By("Waiting for DNS to see that mate and skipper route to service and pod works") + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") err = waitForResponse(hostName, "https", 10*time.Minute, isSuccess, false) Expect(err).NotTo(HaveOccurred()) }) }) + +var __ = framework.KubeDescribe("Ingress tests", func() { + f := framework.NewDefaultFramework("skipper-ingress") + var ( + cs kubernetes.Interface + jig *framework.IngressTestJig + ) + + It("Should create simple ingress [sszuecs] [Ingress] [Zalando]", func() { + jig = framework.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test" + //nameprefix := serviceName + "-" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), e2eHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "mytest" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + waitTime := 10 * time.Minute + + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + deployment, err := cs.Apps().Deployments(ns).Create(depl) + defer func() { + By("deleting the deployment") + defer GinkgoRecover() + err2 := cs.Apps().Deployments(ns).Delete(deployment.Name, metav1.NewDeleteOptions(0)) + Expect(err2).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.Core().Services(ns).Create(service) + Expect(err).NotTo(HaveOccurred()) + + ing := createIngress(serviceName, hostName, ns, labels, port) + ingressCreate, err := cs.Extensions().Ingresses(ns).Create(ing) + Expect(err).NotTo(HaveOccurred()) + + addr, err := jig.WaitForIngressAddress(cs, ns, ingressCreate.Name, waitTime) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.Extensions().Ingresses(ns).Get(ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + Expect(err).NotTo(HaveOccurred()) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + Expect(err).NotTo(HaveOccurred()) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isSuccess, true) + Expect(err).NotTo(HaveOccurred()) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + Expect(err).NotTo(HaveOccurred()) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err := rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + if resp.StatusCode != 200 { + log.Fatalf("Failed to get the right response code from backend: %d", resp.StatusCode) + } + b := make([]byte, 0, 1024) + buf := bytes.NewBuffer(b) + if n, err := io.Copy(buf, resp.Body); err != nil { + log.Fatalf("Failed to copy body: %v", err) + } else { + log.Printf("copy %d bytes of data: %s", n, buf) + } + if s := buf.String(); s != backendContent { + log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) + } + }) +}) diff --git a/test/e2e/util.go b/test/e2e/util.go index cb444ee262..5f0810bf92 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -3,6 +3,7 @@ package e2e import ( "crypto/tls" "fmt" + "net" "net/http" "net/url" "time" @@ -13,8 +14,8 @@ import ( "k8s.io/kubernetes/test/e2e/framework" appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -192,6 +193,54 @@ func createNginxDeploymentWithHostNetwork(nameprefix, namespace, serviceAccount } } +func createSkipperBackendDeployment(nameprefix, namespace, route string, label map[string]string, port, replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: nameprefix + string(uuid.NewUUID()), + Namespace: namespace, + Labels: label, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: label}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: label, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "skipper", + Image: "registry.opensource.zalan.do/pathfinder/skipper:latest", + Args: []string{ + "skipper", + "-inline-routes", + route, + }, + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: port, + }, + }, + Resources: corev1.ResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("250Mi"), + }, + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("250Mi"), + }, + }, + }, + }, + }, + }, + }, + } +} + func createServiceAccount(namespace, serviceAccount string) *v1.ServiceAccount { trueValue := true return &v1.ServiceAccount{ @@ -463,3 +512,27 @@ func createVegetaDeployment(hostPath string, rate int) *appsv1.Deployment { }, } } + +func createHTTPRoundTripper() (http.RoundTripper, chan<- struct{}) { + tr := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + TLSHandshakeTimeout: 5 * time.Second, + IdleConnTimeout: 5 * time.Second, + } + ch := make(chan struct{}) + go func(transport *http.Transport, quit <-chan struct{}) { + for { + select { + case <-time.After(3 * time.Second): + transport.CloseIdleConnections() + case <-quit: + return + } + } + }(tr, ch) + return tr, ch +} From 735274ad5364b4226ef3775544177546a9c9ffc1 Mon Sep 17 00:00:00 2001 From: Alexey Ermakov Date: Thu, 11 Apr 2019 13:59:12 +0200 Subject: [PATCH 25/37] Add oxygen (Compliance monitoring) with read-only access Signed-off-by: Alexey Ermakov --- cluster/node-pools/master-default/userdata.clc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster/node-pools/master-default/userdata.clc.yaml b/cluster/node-pools/master-default/userdata.clc.yaml index c1eb35df61..ff38f274c9 100644 --- a/cluster/node-pools/master-default/userdata.clc.yaml +++ b/cluster/node-pools/master-default/userdata.clc.yaml @@ -469,7 +469,7 @@ storage: - name: TOKEN_INTROSPECTION_URL value: http://127.0.0.1:9021/oauth2/introspect - name: USER_GROUPS - value: credentials-provider=Administrator,credprov-kube-ops-view-read-only-token=ReadOnly,credprov-cluster-lifecycle-manager-cluster-rw-token=Administrator,stups_cluster-lifecycle-manager=Administrator,credprov-cluster-lifecycle-manager-test-cluster-rw-token=Administrator,credprov-cdp-controller-cluster-token=PowerUser,stups_deployment-discovery=ReadOnly,stups_zmon-zmon=ReadOnly,stups_kube-resource-report=ReadOnly,kubelet=system:masters + value: credentials-provider=Administrator,credprov-kube-ops-view-read-only-token=ReadOnly,credprov-cluster-lifecycle-manager-cluster-rw-token=Administrator,stups_cluster-lifecycle-manager=Administrator,credprov-cluster-lifecycle-manager-test-cluster-rw-token=Administrator,credprov-cdp-controller-cluster-token=PowerUser,stups_deployment-discovery=ReadOnly,stups_zmon-zmon=ReadOnly,stups_kube-resource-report=ReadOnly,stups_oxygen=ReadOnly,kubelet=system:masters - name: BUSINESS_PARTNER_IDS value: {{ .Cluster.ConfigItems.apiserver_business_partner_ids }} volumeMounts: From be37f8658aaab79cefc0ad6354976d79a54c39f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 11 Apr 2019 18:16:14 +0200 Subject: [PATCH 26/37] add more ingress test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 155 ++++++++++++++++++++++++++++++++++++++++---- test/e2e/util.go | 74 +++++++++++++++++++++ 2 files changed, 217 insertions(+), 12 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 98dcfd9499..ac4ce7db72 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -14,9 +14,7 @@ limitations under the License. package e2e import ( - "bytes" "fmt" - "io" "log" "net/http" "time" @@ -188,19 +186,152 @@ var __ = framework.KubeDescribe("Ingress tests", func() { Expect(err).NotTo(HaveOccurred()) resp, err := rt.RoundTrip(req) Expect(err).NotTo(HaveOccurred()) - defer resp.Body.Close() + s, err := getBody(resp) + if s != backendContent { + log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) + } + + // Test ingress Predicates with Method("GET") + path := "/" + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-predicate": `Method("GET")`, + }, + port, + ) + ingressUpdate, err := cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with the right content for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) + time.Sleep(10 * time.Second) + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } + + // Test ingress Predicates with Method("PUT") + path = "/" + updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-predicate": `Method("PUT")`, + }, + port, + ) + ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) + time.Sleep(10 * time.Second) + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != 404 { + log.Fatalf("Failed to get the right the right status code 404, got: %d", resp.StatusCode) + } + + // Test ingress Filters + path = "/" + headerKey := "X-Foo" + headerVal := "f00" + updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-filter": fmt.Sprintf(`setResponseHeader("%s", "%s")`, headerKey, headerVal), + }, + port, + ) + ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) + time.Sleep(10 * time.Second) + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) if resp.StatusCode != 200 { - log.Fatalf("Failed to get the right response code from backend: %d", resp.StatusCode) + log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) } - b := make([]byte, 0, 1024) - buf := bytes.NewBuffer(b) - if n, err := io.Copy(buf, resp.Body); err != nil { - log.Fatalf("Failed to copy body: %v", err) - } else { - log.Printf("copy %d bytes of data: %s", n, buf) + if got := resp.Header.Get(headerKey); got != headerVal { + log.Fatalf("Failed to get Header, got: %s, want: %s", got, headerVal) } - if s := buf.String(); s != backendContent { - log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } + + // Test additional hostname + additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), e2eHostedZone()) + addHostIng := addHostIngress(updatedIng, additionalHostname) + ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(addHostIng) + Expect(err).NotTo(HaveOccurred()) + By("Waiting for new DNS hostname to be resolvable " + additionalHostname) + err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) + + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != 200 { + log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) } + By(fmt.Sprintf("Testing the new hostname %s for ingress %s/%s we make sure old routes are working", additionalHostname, ingressUpdate.Namespace, ingressUpdate.Name)) + url = "https://" + additionalHostname + "/" + req, err = http.NewRequest("GET", url, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != 200 { + log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } + + // Test changed path + newPath := "/foo" + changePathIng := changePathIngress(updatedIng, newPath) + ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(changePathIng) + Expect(err).NotTo(HaveOccurred()) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) + time.Sleep(10 * time.Second) + resp, err = rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != 404 { + log.Fatalf("Failed to get the right the right status code 404, got: %d", resp.StatusCode) + } + pathURL := "https://" + hostName + newPath + pathReq, err := http.NewRequest("GET", pathURL, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err = rt.RoundTrip(pathReq) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != 200 { + log.Fatalf("Failed to get the right the right for %s status code 200, got: %d", newPath, resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content from %s after update got: %s, expected: %s", newPath, s, backendContent) + } + }) }) diff --git a/test/e2e/util.go b/test/e2e/util.go index 5f0810bf92..a65438abd5 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1,8 +1,10 @@ package e2e import ( + "bytes" "crypto/tls" "fmt" + "io" "net" "net/http" "net/url" @@ -56,6 +58,65 @@ func createIngress(name, hostname, namespace string, label map[string]string, po } } +func updateIngress(name, namespace, hostname, svcName, path string, labels, annotations map[string]string, port int) *v1beta1.Ingress { + return &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: name, + ServicePort: intstr.FromInt(port), + }, + Rules: []v1beta1.IngressRule{ + { + Host: hostname, + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: path, + Backend: v1beta1.IngressBackend{ + ServiceName: svcName, + ServicePort: intstr.FromInt(port), + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func addHostIngress(ing *v1beta1.Ingress, hostnames ...string) *v1beta1.Ingress { + addRules := []v1beta1.IngressRule{} + origRules := ing.Spec.Rules + + for _, hostname := range hostnames { + for _, rule := range origRules { + r := rule + r.Host = hostname + addRules = append(addRules, r) + } + } + ing.Spec.Rules = append(origRules, addRules...) + return ing +} + +func changePathIngress(ing *v1beta1.Ingress, path string) *v1beta1.Ingress { + for _, rule := range ing.Spec.Rules { + for _, p := range rule.IngressRuleValue.HTTP.Paths { + p.Path = path + } + } + return ing +} + func createNginxDeployment(nameprefix, namespace string, label map[string]string, port, replicas int32) *appsv1.Deployment { zero := int64(0) return &appsv1.Deployment{ @@ -536,3 +597,16 @@ func createHTTPRoundTripper() (http.RoundTripper, chan<- struct{}) { }(tr, ch) return tr, ch } + +func getBody(resp *http.Response) (string, error) { + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return "", fmt.Errorf("response code from backend: %d", resp.StatusCode) + } + b := make([]byte, 0, 1024) + buf := bytes.NewBuffer(b) + if _, err := io.Copy(buf, resp.Body); err != nil { + return "", fmt.Errorf("failed to copy body: %v", err) + } + return buf.String(), nil +} From 9f3f780094b1475f646cdb358d5aaee1d931108e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Thu, 11 Apr 2019 18:17:42 +0200 Subject: [PATCH 27/37] add 4x the current buffer size which is less than 1Mi in total MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- cluster/config-defaults.yaml | 1 + cluster/manifests/skipper/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml index c770f1bb93..45dc871efa 100644 --- a/cluster/config-defaults.yaml +++ b/cluster/config-defaults.yaml @@ -23,6 +23,7 @@ skipper_ingress_max_replicas: "30" skipper_ingress_min_replicas: "3" skipper_ingress_cpu: "500m" skipper_ingress_memory: "500Mi" +skipper_ingress_tracing_buffer: "16384" # skipper backend timeout defaults skipper_expect_continue_timeout_backend: "30s" diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml index 235347c255..30b57fc24f 100644 --- a/cluster/manifests/skipper/deployment.yaml +++ b/cluster/manifests/skipper/deployment.yaml @@ -89,7 +89,7 @@ spec: {{ end }} - "-oauth2-tokeninfo-url={{ .ConfigItems.tokeninfo_url }}" - "-histogram-metric-buckets=.0001,.00025,.0005,.00075,.001,.0025,.005,.0075,.01,.025,.05,.075,.1,.2,.3,.4,.5,.75,1,2,3,4,5,7,10,15,20,30,60,120,300,600" - - "-opentracing=lightstep component-name=skipper-ingress token=$(LIGHTSTEP_TOKEN) collector=tracing.stups.zalan.do:8444 cmd-line=skipper-ingress max-buffered-spans=4096" + - "-opentracing=lightstep component-name=skipper-ingress token=$(LIGHTSTEP_TOKEN) collector=tracing.stups.zalan.do:8444 cmd-line=skipper-ingress max-buffered-spans={{ .ConfigItems.skipper_ingress_tracing_buffer }}" - "-expect-continue-timeout-backend={{ .ConfigItems.skipper_expect_continue_timeout_backend }}" - "-keepalive-backend={{ .ConfigItems.skipper_keepalive_backend }}" - "-max-idle-connection-backend={{ .ConfigItems.skipper_max_idle_connection_backend }}" From 917d1fed131c2479e65a676a6eacd9b798d68037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 15:50:57 +0200 Subject: [PATCH 28/37] add some helpers to make tests faster and more reliable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 47 ++++++++++++++++++---------------- test/e2e/util.go | 62 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 37 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index ac4ce7db72..e602adee86 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -99,7 +99,7 @@ var _ = framework.KubeDescribe("Ingress ALB creation", func() { // ALB ready By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") - err = waitForResponse(addr, "https", 10*time.Minute, isSuccess, true) + err = waitForResponse(addr, "https", 10*time.Minute, isNotFound, true) Expect(err).NotTo(HaveOccurred()) // DNS ready @@ -120,7 +120,6 @@ var __ = framework.KubeDescribe("Ingress tests", func() { jig = framework.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" - //nameprefix := serviceName + "-" ns := f.Namespace.Name hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), e2eHostedZone()) labels := map[string]string{ @@ -133,6 +132,7 @@ var __ = framework.KubeDescribe("Ingress tests", func() { route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) waitTime := 10 * time.Minute + // CREATE setup // backend deployment By("Creating a deployment with " + serviceName + " in namespace " + ns) depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) @@ -167,7 +167,7 @@ var __ = framework.KubeDescribe("Ingress tests", func() { // ALB ready By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") - err = waitForResponse(addr, "https", waitTime, isSuccess, true) + err = waitForResponse(addr, "https", waitTime, isNotFound, true) Expect(err).NotTo(HaveOccurred()) // DNS ready @@ -187,10 +187,12 @@ var __ = framework.KubeDescribe("Ingress tests", func() { resp, err := rt.RoundTrip(req) Expect(err).NotTo(HaveOccurred()) s, err := getBody(resp) + Expect(err).NotTo(HaveOccurred()) if s != backendContent { log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) } + // Start actual ingress tests // Test ingress Predicates with Method("GET") path := "/" updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, @@ -207,9 +209,11 @@ var __ = framework.KubeDescribe("Ingress tests", func() { ingressUpdate, err := cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with the right content for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) - time.Sleep(10 * time.Second) - resp, err = rt.RoundTrip(req) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusOK { + log.Fatalf("Failed to get status code expected status code 200: %d", resp.StatusCode) + } s, err = getBody(resp) Expect(err).NotTo(HaveOccurred()) if s != backendContent { @@ -232,10 +236,9 @@ var __ = framework.KubeDescribe("Ingress tests", func() { ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) - time.Sleep(10 * time.Second) - resp, err = rt.RoundTrip(req) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 404 { + if resp.StatusCode != http.StatusNotFound { log.Fatalf("Failed to get the right the right status code 404, got: %d", resp.StatusCode) } @@ -257,10 +260,10 @@ var __ = framework.KubeDescribe("Ingress tests", func() { ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) - time.Sleep(10 * time.Second) - resp, err = rt.RoundTrip(req) + time.Sleep(10 * time.Second) // wait for routing change propagation + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) } if got := resp.Header.Get(headerKey); got != headerVal { @@ -281,10 +284,9 @@ var __ = framework.KubeDescribe("Ingress tests", func() { err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) Expect(err).NotTo(HaveOccurred()) By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) - - resp, err = rt.RoundTrip(req) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) } s, err = getBody(resp) @@ -296,9 +298,9 @@ var __ = framework.KubeDescribe("Ingress tests", func() { url = "https://" + additionalHostname + "/" req, err = http.NewRequest("GET", url, nil) Expect(err).NotTo(HaveOccurred()) - resp, err = rt.RoundTrip(req) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to get the right the right status code 200, got: %d", resp.StatusCode) } s, err = getBody(resp) @@ -312,19 +314,20 @@ var __ = framework.KubeDescribe("Ingress tests", func() { changePathIng := changePathIngress(updatedIng, newPath) ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(changePathIng) Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the next request", ingressUpdate.Namespace, ingressUpdate.Name)) - time.Sleep(10 * time.Second) - resp, err = rt.RoundTrip(req) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the old request, because of the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 404 { + if resp.StatusCode != http.StatusNotFound { log.Fatalf("Failed to get the right the right status code 404, got: %d", resp.StatusCode) } pathURL := "https://" + hostName + newPath pathReq, err := http.NewRequest("GET", pathURL, nil) Expect(err).NotTo(HaveOccurred()) - resp, err = rt.RoundTrip(pathReq) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 for a new request to the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, pathReq, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { log.Fatalf("Failed to get the right the right for %s status code 200, got: %d", newPath, resp.StatusCode) } s, err = getBody(resp) diff --git a/test/e2e/util.go b/test/e2e/util.go index a65438abd5..ff61ff0f0a 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "io" + "log" "net" "net/http" "net/url" @@ -32,10 +33,6 @@ func createIngress(name, hostname, namespace string, label map[string]string, po Labels: label, }, Spec: v1beta1.IngressSpec{ - Backend: &v1beta1.IngressBackend{ - ServiceName: name, - ServicePort: intstr.FromInt(port), - }, Rules: []v1beta1.IngressRule{ { Host: hostname, @@ -67,10 +64,6 @@ func updateIngress(name, namespace, hostname, svcName, path string, labels, anno Annotations: annotations, }, Spec: v1beta1.IngressSpec{ - Backend: &v1beta1.IngressBackend{ - ServiceName: name, - ServicePort: intstr.FromInt(port), - }, Rules: []v1beta1.IngressRule{ { Host: hostname, @@ -109,12 +102,16 @@ func addHostIngress(ing *v1beta1.Ingress, hostnames ...string) *v1beta1.Ingress } func changePathIngress(ing *v1beta1.Ingress, path string) *v1beta1.Ingress { - for _, rule := range ing.Spec.Rules { - for _, p := range rule.IngressRuleValue.HTTP.Paths { - p.Path = path - } - } - return ing + return updateIngress( + ing.ObjectMeta.Name, + ing.ObjectMeta.Namespace, + ing.Spec.Rules[0].Host, + ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.ServiceName, + path, + ing.ObjectMeta.Labels, + ing.ObjectMeta.Annotations, + ing.Spec.Rules[0].IngressRuleValue.HTTP.Paths[0].Backend.ServicePort.IntValue(), + ) } func createNginxDeployment(nameprefix, namespace string, label map[string]string, port, replicas int32) *appsv1.Deployment { @@ -428,7 +425,11 @@ func isRedirect(code int) bool { } func isSuccess(code int) bool { - return code == 200 + return code == http.StatusOK +} + +func isNotFound(code int) bool { + return code == http.StatusNotFound } func waitForResponse(hostname, scheme string, timeout time.Duration, expectedCode func(int) bool, insecure bool) error { @@ -598,6 +599,37 @@ func createHTTPRoundTripper() (http.RoundTripper, chan<- struct{}) { return tr, ch } +func getAndWaitResponse(rt http.RoundTripper, req *http.Request, timeout time.Duration, expectedStatusCode int) (resp *http.Response, err error) { + d := 1 * time.Second + if timeout < d { + d = timeout - 1 + } + timeoutCH := make(chan struct{}) + go func() { + time.Sleep(timeout) + timeoutCH <- struct{}{} + }() + + for { + resp, err = rt.RoundTrip(req) + if err == nil && resp.StatusCode == expectedStatusCode { + return + } + if err != nil { + log.Printf("Failed to do rountrip: %v", err) + } + + select { + case <-timeoutCH: + log.Printf("timeout to GET %s", req.URL) + return + case <-time.After(d): + log.Printf("retry to GET %s", req.URL) + continue + } + } +} + func getBody(resp *http.Response) (string, error) { defer resp.Body.Close() if resp.StatusCode >= 400 { From c4c0f257352bfe1a6dd95b5cffd9a3afeb09c2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 15:57:06 +0200 Subject: [PATCH 29/37] remove tag filter used for local testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index e602adee86..01236693d1 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -116,7 +116,7 @@ var __ = framework.KubeDescribe("Ingress tests", func() { jig *framework.IngressTestJig ) - It("Should create simple ingress [sszuecs] [Ingress] [Zalando]", func() { + It("Should create simple ingress [Ingress] [Zalando]", func() { jig = framework.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" From 151ea00ee1c6115fe45b2021954bbb7946a6890a Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 12 Apr 2019 17:38:02 +0200 Subject: [PATCH 30/37] Switch to redis based statefulset test CockroachDB statefulset test is very flaky, try with redis based test instead. Signed-off-by: Mikkel Oscar Lyderik Larsen --- test/e2e/README.md | 2 +- test/e2e/run_e2e.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 178beb2d11..09fbbecf50 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -26,7 +26,7 @@ examples of how to write the tests or checkout the files already defined e.g. ```bash KUBECONFIG=~/.kube/config HOSTED_ZONE=example.org \ ginkgo -nodes=25 -flakeAttempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*CockroachDB|\[Zalando\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*redis|\[Zalando\])" \ -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false ``` diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index ca07ef2aef..85eafd2ee7 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -104,7 +104,7 @@ export KUBECONFIG="$KUBECONFIG" # * statefulset tests # * custom 'zalando' tests ginkgo -nodes=25 -flakeAttempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*CockroachDB|\[Zalando\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*redis|\[Zalando\])" \ -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false From 6f54d55dfbb0f6a9894348ec9cfab3e52130707d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 17:51:01 +0200 Subject: [PATCH 31/37] add path test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 180 +++++++++++++++++++++++++++++++++++++++++++- test/e2e/util.go | 19 +++++ 2 files changed, 197 insertions(+), 2 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 01236693d1..726da58ba0 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -22,7 +22,9 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" ) @@ -109,8 +111,8 @@ var _ = framework.KubeDescribe("Ingress ALB creation", func() { }) }) -var __ = framework.KubeDescribe("Ingress tests", func() { - f := framework.NewDefaultFramework("skipper-ingress") +var __ = framework.KubeDescribe("Ingress tests simple", func() { + f := framework.NewDefaultFramework("skipper-ingress simple") var ( cs kubernetes.Interface jig *framework.IngressTestJig @@ -338,3 +340,177 @@ var __ = framework.KubeDescribe("Ingress tests", func() { }) }) + +var ___ = framework.KubeDescribe("Ingress tests paths", func() { + f := framework.NewDefaultFramework("skipper-ingress paths") + var ( + cs kubernetes.Interface + jig *framework.IngressTestJig + ) + + It("Should create complex path routes ingress [sszuecs] [Ingress] [Zalando]", func() { + jig = framework.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test-pr" + serviceName2 := "skipper-ingress-test-pr2" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), e2eHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + labels2 := map[string]string{ + "app": serviceName2, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "be-foo" + backendContent2 := "be-bar" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + route2 := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent2) + waitTime := 10 * time.Minute + + // CREATE setup + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + deployment, err := cs.Apps().Deployments(ns).Create(depl) + defer func() { + By("deleting the deployment") + defer GinkgoRecover() + err2 := cs.Apps().Deployments(ns).Delete(deployment.Name, metav1.NewDeleteOptions(0)) + Expect(err2).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + By("Creating a 2nd deployment with " + serviceName2 + " in namespace " + ns) + depl2 := createSkipperBackendDeployment(serviceName2, ns, route2, labels2, int32(targetPort), replicas) + deployment2, err := cs.Apps().Deployments(ns).Create(depl2) + defer func() { + By("deleting the deployment") + defer GinkgoRecover() + err2 := cs.Apps().Deployments(ns).Delete(deployment2.Name, metav1.NewDeleteOptions(0)) + Expect(err2).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.Core().Services(ns).Create(service) + Expect(err).NotTo(HaveOccurred()) + + By("Creating service " + serviceName2 + " in namespace " + ns) + service2 := createServiceTypeClusterIP(serviceName2, labels2, port, targetPort) + _, err = cs.Core().Services(ns).Create(service2) + Expect(err).NotTo(HaveOccurred()) + + By("Creating ingress " + serviceName + " in namespace " + ns + "with hostname " + hostName) + ing := createIngress(serviceName, hostName, ns, labels, port) + ingressCreate, err := cs.Extensions().Ingresses(ns).Create(ing) + Expect(err).NotTo(HaveOccurred()) + + addr, err := jig.WaitForIngressAddress(cs, ns, ingressCreate.Name, waitTime) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.Extensions().Ingresses(ns).Get(ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + Expect(err).NotTo(HaveOccurred()) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + Expect(err).NotTo(HaveOccurred()) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + Expect(err).NotTo(HaveOccurred()) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + Expect(err).NotTo(HaveOccurred()) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err := rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + s, err := getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) + } + + // Start actual ingress tests + // Test ingress with 1 path + bepath := "/foo" + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + bepath, + ingressCreate.ObjectMeta.Labels, + ingressCreate.ObjectMeta.Annotations, + port, + ) + ingressUpdate, err := cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 404 for path /", ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusNotFound { + log.Fatalf("Failed to get status code expected status code 404: %d", resp.StatusCode) + } + + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 200 for path %s", ingressUpdate.Namespace, ingressUpdate.Name, bepath)) + beurl := "https://" + hostName + bepath + bereq, err := http.NewRequest("GET", beurl, nil) + resp, err = getAndWaitResponse(rt, bereq, 10*time.Second, http.StatusOK) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusOK { + log.Fatalf("Failed to get status code expected status code 200: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } + + // Test ingress with 2 paths + bepath2 := "/bar" + beurl2 := "https://" + hostName + bepath2 + bereq2, err := http.NewRequest("GET", beurl2, nil) + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 404 for path %s", ingressUpdate.Namespace, ingressUpdate.Name, bepath2)) + resp, err = getAndWaitResponse(rt, bereq2, 10*time.Second, http.StatusNotFound) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusNotFound { + log.Fatalf("Failed to get status code expected status code 404: %d", resp.StatusCode) + } + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 200 for path %s", ingressUpdate.Namespace, ingressUpdate.Name, bepath2)) + updatedIng = addPathIngress(updatedIng, + bepath2, + v1beta1.IngressBackend{ + ServiceName: serviceName2, + ServicePort: intstr.FromInt(port), + }, + ) + ingressUpdate, err = cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + resp, err = getAndWaitResponse(rt, bereq2, 10*time.Second, http.StatusOK) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusOK { + log.Fatalf("Failed to get status code expected status code 200: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent2 { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent2) + } + }) +}) diff --git a/test/e2e/util.go b/test/e2e/util.go index ff61ff0f0a..3f6fe0fbe0 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -101,6 +101,25 @@ func addHostIngress(ing *v1beta1.Ingress, hostnames ...string) *v1beta1.Ingress return ing } +func addPathIngress(ing *v1beta1.Ingress, path string, backend v1beta1.IngressBackend) *v1beta1.Ingress { + addRules := []v1beta1.IngressRule{} + origRules := ing.Spec.Rules + + for _, rule := range origRules { + r := rule + r.Host = rule.Host + origPaths := r.IngressRuleValue.HTTP.Paths + origPaths = append(origPaths, v1beta1.HTTPIngressPath{ + Path: path, + Backend: backend, + }) + r.IngressRuleValue.HTTP.Paths = origPaths + addRules = append(addRules, r) + } + ing.Spec.Rules = addRules + return ing +} + func changePathIngress(ing *v1beta1.Ingress, path string) *v1beta1.Ingress { return updateIngress( ing.ObjectMeta.Name, From be2f87bd7e5bf07a3dacfc80229cf1b2e911b588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 18:00:00 +0200 Subject: [PATCH 32/37] fix naming convention and test former path route to be ok also after added the second path route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 726da58ba0..02e6cce2c1 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -112,7 +112,7 @@ var _ = framework.KubeDescribe("Ingress ALB creation", func() { }) var __ = framework.KubeDescribe("Ingress tests simple", func() { - f := framework.NewDefaultFramework("skipper-ingress simple") + f := framework.NewDefaultFramework("skipper-ingress-simple") var ( cs kubernetes.Interface jig *framework.IngressTestJig @@ -342,13 +342,13 @@ var __ = framework.KubeDescribe("Ingress tests simple", func() { }) var ___ = framework.KubeDescribe("Ingress tests paths", func() { - f := framework.NewDefaultFramework("skipper-ingress paths") + f := framework.NewDefaultFramework("skipper-ingress-paths") var ( cs kubernetes.Interface jig *framework.IngressTestJig ) - It("Should create complex path routes ingress [sszuecs] [Ingress] [Zalando]", func() { + It("Should create path routes ingress [sszuecs] [Ingress] [Zalando]", func() { jig = framework.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test-pr" @@ -512,5 +512,19 @@ var ___ = framework.KubeDescribe("Ingress tests paths", func() { if s != backendContent2 { log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent2) } + + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 200 for path %s without change from the other path", ingressUpdate.Namespace, ingressUpdate.Name, bepath)) + beurl := "https://" + hostName + bepath + bereq, err := http.NewRequest("GET", beurl, nil) + resp, err = getAndWaitResponse(rt, bereq, 10*time.Second, http.StatusOK) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusOK { + log.Fatalf("Failed to get status code expected status code 200: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } }) }) From 1ee57af749105ffd5b7fb60a55806adf76166b76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 18:00:29 +0200 Subject: [PATCH 33/37] remove test tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 02e6cce2c1..53871f4536 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -348,7 +348,7 @@ var ___ = framework.KubeDescribe("Ingress tests paths", func() { jig *framework.IngressTestJig ) - It("Should create path routes ingress [sszuecs] [Ingress] [Zalando]", func() { + It("Should create path routes ingress [Ingress] [Zalando]", func() { jig = framework.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test-pr" From c2c92c457fe1dcd005c90a3de42bbbddc2480e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Fri, 12 Apr 2019 19:14:21 +0200 Subject: [PATCH 34/37] add custom routes test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/ingress.go | 138 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 2 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 53871f4536..b19775bcba 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -514,8 +514,8 @@ var ___ = framework.KubeDescribe("Ingress tests paths", func() { } By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 200 for path %s without change from the other path", ingressUpdate.Namespace, ingressUpdate.Name, bepath)) - beurl := "https://" + hostName + bepath - bereq, err := http.NewRequest("GET", beurl, nil) + beurl = "https://" + hostName + bepath + bereq, err = http.NewRequest("GET", beurl, nil) resp, err = getAndWaitResponse(rt, bereq, 10*time.Second, http.StatusOK) Expect(err).NotTo(HaveOccurred()) if resp.StatusCode != http.StatusOK { @@ -528,3 +528,137 @@ var ___ = framework.KubeDescribe("Ingress tests paths", func() { } }) }) + +var ____ = framework.KubeDescribe("Ingress tests custom routes", func() { + f := framework.NewDefaultFramework("skipper-ingress-custom") + var ( + cs kubernetes.Interface + jig *framework.IngressTestJig + ) + + It("Should create custom routes ingress [Ingress] [Zalando]", func() { + jig = framework.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test-custom" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), e2eHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "custom-foo" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + waitTime := 10 * time.Minute + + // CREATE setup + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + deployment, err := cs.Apps().Deployments(ns).Create(depl) + defer func() { + By("deleting the deployment") + defer GinkgoRecover() + err2 := cs.Apps().Deployments(ns).Delete(deployment.Name, metav1.NewDeleteOptions(0)) + Expect(err2).NotTo(HaveOccurred()) + }() + Expect(err).NotTo(HaveOccurred()) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.Core().Services(ns).Create(service) + Expect(err).NotTo(HaveOccurred()) + + By("Creating ingress " + serviceName + " in namespace " + ns + "with hostname " + hostName) + ing := createIngress(serviceName, hostName, ns, labels, port) + ingressCreate, err := cs.Extensions().Ingresses(ns).Create(ing) + Expect(err).NotTo(HaveOccurred()) + + addr, err := jig.WaitForIngressAddress(cs, ns, ingressCreate.Name, waitTime) + Expect(err).NotTo(HaveOccurred()) + + _, err = cs.Extensions().Ingresses(ns).Get(ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + Expect(err).NotTo(HaveOccurred()) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + Expect(err).NotTo(HaveOccurred()) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + Expect(err).NotTo(HaveOccurred()) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + Expect(err).NotTo(HaveOccurred()) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err := rt.RoundTrip(req) + Expect(err).NotTo(HaveOccurred()) + s, err := getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content got: %s, expected: %s", s, backendContent) + } + + // Start actual ingress tests + // Test ingress with 1 custom route + path := "/" + baseURL := "https://" + hostName + redirectDestinationURL := baseURL + path + redirectPath := "/redirect" + redirectURL := baseURL + redirectPath + redirectRoute := fmt.Sprintf(`redirecttoself: PathRegexp("%s") -> modPath("%s", "%s") -> redirectTo(307, "%s") -> ;`, redirectPath, redirectPath, path, redirectDestinationURL) + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-routes": redirectRoute, + }, + port, + ) + ingressUpdate, err := cs.Extensions().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(updatedIng) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Testing for ingress %s/%s we want to get a 307 for path %s", ingressUpdate.Namespace, ingressUpdate.Name, redirectPath)) + req, err = http.NewRequest("GET", redirectURL, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusTemporaryRedirect) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusTemporaryRedirect { + log.Fatalf("Failed to get status code expected status code 307: %d", resp.StatusCode) + } + + reqRedirectURL := resp.Header.Get("Location") + By(fmt.Sprintf("Testing for ingress %s/%s rediretc Location we want to get a 200 for URL %s", ingressUpdate.Namespace, ingressUpdate.Name, reqRedirectURL)) + if redirectDestinationURL != reqRedirectURL { + log.Fatalf("Failed to get the right redirect from header: %s, expected: %s", reqRedirectURL, redirectDestinationURL) + } + redirectreq, err := http.NewRequest("GET", reqRedirectURL, nil) + resp, err = getAndWaitResponse(rt, redirectreq, 10*time.Second, http.StatusOK) + Expect(err).NotTo(HaveOccurred()) + if resp.StatusCode != http.StatusOK { + log.Fatalf("Failed to get status code expected status code 200: %d", resp.StatusCode) + } + s, err = getBody(resp) + Expect(err).NotTo(HaveOccurred()) + if s != backendContent { + log.Fatalf("Failed to get the right content after update got: %s, expected: %s", s, backendContent) + } + }) +}) From 94ea82ff852e1c3ea324794fcbcd742248aeac6f Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 12 Apr 2019 19:17:04 +0200 Subject: [PATCH 35/37] Use mysql statefulset tests instead of redis Signed-off-by: Mikkel Oscar Lyderik Larsen --- test/e2e/README.md | 2 +- test/e2e/run_e2e.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 09fbbecf50..ec7a9d2296 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -26,7 +26,7 @@ examples of how to write the tests or checkout the files already defined e.g. ```bash KUBECONFIG=~/.kube/config HOSTED_ZONE=example.org \ ginkgo -nodes=25 -flakeAttempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*redis|\[Zalando\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false ``` diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 85eafd2ee7..005948df19 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -104,7 +104,7 @@ export KUBECONFIG="$KUBECONFIG" # * statefulset tests # * custom 'zalando' tests ginkgo -nodes=25 -flakeAttempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*redis|\[Zalando\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ -skip="(\[Serial\])" \ "e2e.test" -- -delete-namespace-on-failure=false From 74ea3442ce9f2bd6f93cf7df7e3929c0a94f4e74 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 12 Apr 2019 22:28:30 +0200 Subject: [PATCH 36/37] Update metrics-server to v0.3.2, use RBAC Signed-off-by: Mikkel Oscar Lyderik Larsen --- .../manifests/metrics-server/deployment.yaml | 24 +++++--- cluster/manifests/metrics-server/rbac.yaml | 61 +++++++++++++++++++ 2 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 cluster/manifests/metrics-server/rbac.yaml diff --git a/cluster/manifests/metrics-server/deployment.yaml b/cluster/manifests/metrics-server/deployment.yaml index 138e61bd89..426173a48a 100644 --- a/cluster/manifests/metrics-server/deployment.yaml +++ b/cluster/manifests/metrics-server/deployment.yaml @@ -5,7 +5,7 @@ metadata: namespace: kube-system labels: application: metrics-server - version: v0.2.1 + version: v0.3.2 spec: replicas: 1 selector: @@ -16,20 +16,23 @@ spec: name: metrics-server labels: application: metrics-server - version: v0.2.1 + version: v0.3.2 spec: dnsConfig: options: - name: ndots value: "1" priorityClassName: system-cluster-critical - serviceAccountName: system + serviceAccountName: metrics-server containers: - name: metrics-server - image: registry.opensource.zalan.do/teapot/metrics-server:v0.2.1 - command: - - /metrics-server - - --source=kubernetes.summary_api:'' + image: registry.opensource.zalan.do/teapot/metrics-server:v0.3.2 + args: + # Connect to kubelet on 'completely insecure' port. + # We need to configure kubelet differently to be able to use the secure + # port 10250. + - --deprecated-kubelet-completely-insecure + - --kubelet-port=10255 resources: limits: cpu: "{{.ConfigItems.metrics_service_cpu}}" @@ -37,3 +40,10 @@ spec: requests: cpu: "{{.ConfigItems.metrics_service_cpu}}" memory: "{{.ConfigItems.metrics_service_mem}}" + volumeMounts: + - name: tmp-dir + mountPath: /tmp + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} diff --git a/cluster/manifests/metrics-server/rbac.yaml b/cluster/manifests/metrics-server/rbac.yaml new file mode 100644 index 0000000000..8b9d9c567f --- /dev/null +++ b/cluster/manifests/metrics-server/rbac.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system From 229d93e1e6c45177d961520afe18e2f1aec96611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandor=20Sz=C3=BCcs?= Date: Mon, 15 Apr 2019 11:40:18 +0200 Subject: [PATCH 37/37] fix image policy not allowing latest for skipper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sandor Szücs --- test/e2e/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/util.go b/test/e2e/util.go index 3f6fe0fbe0..1e86a50ebd 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -288,7 +288,7 @@ func createSkipperBackendDeployment(nameprefix, namespace, route string, label m Containers: []corev1.Container{ { Name: "skipper", - Image: "registry.opensource.zalan.do/pathfinder/skipper:latest", + Image: "registry.opensource.zalan.do/pathfinder/skipper:v0.10.203", Args: []string{ "skipper", "-inline-routes",