From dee7728170a423ed388c3275524e6e116d746253 Mon Sep 17 00:00:00 2001 From: Yang Yang Date: Thu, 12 Mar 2020 10:47:23 -0700 Subject: [PATCH] add e2e tests --- go.mod | 4 + go.sum | 30 +- test/e2e/fishapp/dynamic_stack.go | 811 ++++++++++++++++++ test/e2e/fishapp/dynamic_stack_test.go | 66 ++ test/e2e/fishapp/fishapp_suite_test.go | 13 + test/e2e/fishapp/shared/manifest_builder.go | 275 ++++++ test/e2e/framework/collection/map.go | 11 + test/e2e/framework/framework.go | 75 ++ test/e2e/framework/k8s/patch.go | 37 + test/e2e/framework/k8s/port_forward.go | 33 + test/e2e/framework/options.go | 43 + .../framework/resource/deployment/manager.go | 61 ++ test/e2e/framework/resource/mesh/manager.go | 44 + .../framework/resource/namespace/manager.go | 72 ++ .../framework/resource/virtualnode/manager.go | 60 ++ .../resource/virtualservice/manager.go | 69 ++ test/e2e/framework/utils/log.go | 25 + test/e2e/framework/utils/name.go | 18 + test/e2e/framework/utils/poll.go | 8 + 19 files changed, 1727 insertions(+), 28 deletions(-) create mode 100644 test/e2e/fishapp/dynamic_stack.go create mode 100644 test/e2e/fishapp/dynamic_stack_test.go create mode 100644 test/e2e/fishapp/fishapp_suite_test.go create mode 100644 test/e2e/fishapp/shared/manifest_builder.go create mode 100644 test/e2e/framework/collection/map.go create mode 100644 test/e2e/framework/framework.go create mode 100644 test/e2e/framework/k8s/patch.go create mode 100644 test/e2e/framework/k8s/port_forward.go create mode 100644 test/e2e/framework/options.go create mode 100644 test/e2e/framework/resource/deployment/manager.go create mode 100644 test/e2e/framework/resource/mesh/manager.go create mode 100644 test/e2e/framework/resource/namespace/manager.go create mode 100644 test/e2e/framework/resource/virtualnode/manager.go create mode 100644 test/e2e/framework/resource/virtualservice/manager.go create mode 100644 test/e2e/framework/utils/log.go create mode 100644 test/e2e/framework/utils/name.go create mode 100644 test/e2e/framework/utils/poll.go diff --git a/go.mod b/go.mod index e248b243d..1ebfccf79 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,13 @@ go 1.13 require ( github.com/aws/aws-sdk-go v1.29.13 github.com/deckarep/golang-set v1.7.1 + github.com/evanphx/json-patch v4.2.0+incompatible github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/googleapis/gnostic v0.2.0 // indirect github.com/imdario/mergo v0.3.7 // indirect + github.com/onsi/ginkgo v1.10.1 + github.com/onsi/gomega v1.7.0 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/spf13/cobra v0.0.5 diff --git a/go.sum b/go.sum index b9ace201a..482a70c9b 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -11,7 +10,6 @@ github.com/aws/aws-sdk-go v1.29.13 h1:Y77U33nj5ic5hVxE6Th4LhZaw2rSwl3mXIm9OdmIs+ github.com/aws/aws-sdk-go v1.29.13/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -23,6 +21,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -45,7 +44,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -72,7 +70,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -81,10 +78,8 @@ github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62F github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -112,8 +107,8 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -128,7 +123,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nL github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -140,7 +134,6 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -151,28 +144,22 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 h1:Xim2mBRFdXzXmKRO8DJg/FJtn/8Fj9NOEpO6+WuMPmk= github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5/go.mod h1:ppEjwdhyy7Y31EnHRDm1JkChoC7LXIJ7Ex0VYLWtZtQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 h1:a4tQYYYuK9QdeO/+kEvNYyuR21S+7ve5EANok6hABhI= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495 h1:I6A9Ag9FpEKOjcKrRNjQkPHawoXIhKyTGfvvjFAiiAk= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -182,9 +169,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -193,7 +178,6 @@ golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -216,21 +200,14 @@ golang.org/x/tools v0.0.0-20181112210238-4b1f3b6b1646/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20200212213342-7a21e308cf6c h1:D2X+P0Z6ychko7xn2jvd38yxQfdU0eksO4AHfd8AWFI= golang.org/x/tools v0.0.0-20200212213342-7a21e308cf6c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -240,7 +217,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -250,11 +226,9 @@ k8s.io/apimachinery v0.0.0-20191025225532-af6325b3a843 h1:Ge6Np+ecN6pKYcAaFXR53I k8s.io/apimachinery v0.0.0-20191025225532-af6325b3a843/go.mod h1:gA1T9z4LIup7PIegBwxkF2UYXUNVKhOAPvQWWnAc34k= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= -k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b h1:p+PRuwXWwk5e+UYvicGiavEupapqM5NOxUl3y1GkD6c= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= diff --git a/test/e2e/fishapp/dynamic_stack.go b/test/e2e/fishapp/dynamic_stack.go new file mode 100644 index 000000000..65c3de075 --- /dev/null +++ b/test/e2e/fishapp/dynamic_stack.go @@ -0,0 +1,811 @@ +package fishapp + +import ( + "context" + "encoding/json" + "fmt" + appmeshv1beta1 "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/apis/appmesh/v1beta1" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/fishapp/shared" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/collection" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/k8s" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/servicediscovery" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + "io/ioutil" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "net/http" + "net/url" + "sync" + "time" +) + +const ( + connectivityCheckRate = time.Second / 100 + appMeshOperationRate = time.Second * 2 +) + +// A dynamic generated stack designed to test app mesh integration :D +// Suppose we have configuration: +// 5 VirtualServicesCount +// 10 VirtualNodesCount +// 2 RoutesCountPerVirtualRouter +// 2 TargetsCountPerRoute +// 4 BackendsCountPerVirtualNode +// we will generate below virtual service configuration & virtual node configuration +// =======virtual services ========= +// vs1 -> /path1 -> vn1(50) +// -> vn2(50) +// -> /path2 -> vn3(50) +// -> vn4(50) +// vs2 -> /path1 -> vn5(50) +// -> vn6(50) +// -> /path2 -> vn7(50) +// -> vn8(50) +// vs3 -> /path1 -> vn9(50) +// -> vn10(50) +// -> /path2 -> vn1(50) +// -> vn2(50) +// vs4 -> /path1 -> vn3(50) +// -> vn4(50) +// -> /path2 -> vn5(50) +// -> vn6(50) +// vs5 -> /path1 -> vn7(50) +// -> vn8(50) +// -> /path2 -> vn9(50) +// -> vn10(50) +// =======virtual nodes ========= +// vn1 -> vs1,vs2,vs3,vs4 +// vn2 -> vs5,vs1,vs2,vs3 +// vn3 -> vs4,vs5,vs1,vs2 +// ... +// +// then we validate each virtual node can access each virtual service at every path, and calculates the target distribution +type DynamicStack struct { + // service discovery type + ServiceDiscoveryType shared.ServiceDiscoveryType + + // number of virtual service + VirtualServicesCount int + + // number of virtual nodes count + VirtualNodesCount int + + // number of routes per virtual router + RoutesCountPerVirtualRouter int + + // number of targets per route + TargetsCountPerRoute int + + // number of backends per virtual node + BackendsCountPerVirtualNode int + + // number of replicas per virtual node + ReplicasPerVirtualNode int32 + + // how many time to check connectivity per URL + ConnectivityCheckPerURL int + + // ====== runtime variables ====== + mesh *appmeshv1beta1.Mesh + namespace *corev1.Namespace + cloudMapNamespace string + + createdNodeVNs []*appmeshv1beta1.VirtualNode + createdNodeDPs []*appsv1.Deployment + createdNodeSVCs []*corev1.Service + + createdServiceVSs []*appmeshv1beta1.VirtualService + createdServiceSVCs []*corev1.Service +} + +// expects the stack can be deployed to namespace successfully +func (s *DynamicStack) Deploy(ctx context.Context, f *framework.Framework) { + s.createMeshAndNamespace(ctx, f) + if s.ServiceDiscoveryType == shared.CloudMapServiceDiscovery { + s.createCloudMapNamespace(ctx, f) + } + mb := &shared.ManifestBuilder{ + MeshName: s.mesh.Name, + Namespace: s.namespace.Name, + ServiceDiscoveryType: s.ServiceDiscoveryType, + CloudMapNamespace: s.cloudMapNamespace, + } + s.createResourcesForNodes(ctx, f, mb) + s.createResourcesForServices(ctx, f, mb) + s.grantVirtualNodesBackendAccess(ctx, f) +} + +// expects the stack can be cleaned up from namespace successfully +func (s *DynamicStack) Cleanup(ctx context.Context, f *framework.Framework) { + var deletionErrors []error + if errs := s.revokeVirtualNodeBackendAccess(ctx, f); len(errs) != 0 { + deletionErrors = append(deletionErrors, errs...) + } + if errs := s.deleteResourcesForServices(ctx, f); len(errs) != 0 { + deletionErrors = append(deletionErrors, errs...) + } + if errs := s.deleteResourcesForNodes(ctx, f); len(errs) != 0 { + deletionErrors = append(deletionErrors, errs...) + } + if s.ServiceDiscoveryType == shared.CloudMapServiceDiscovery { + if errs := s.deleteCloudMapNamespace(ctx, f); len(errs) != 0 { + deletionErrors = append(deletionErrors, errs...) + } + } + if errs := s.deleteMeshAndNamespace(ctx, f); len(errs) != 0 { + deletionErrors = append(deletionErrors, errs...) + } + Expect(len(deletionErrors)).To(BeZero()) +} + +// expects connectivity and routing works correctly +func (s *DynamicStack) ExpectBehavesCorrectly(ctx context.Context, f *framework.Framework) { + time.Sleep(3 * time.Minute) + vsByName := make(map[string]*appmeshv1beta1.VirtualService) + for i := 0; i != s.VirtualServicesCount; i++ { + vs := s.createdServiceVSs[i] + vsByName[vs.Name] = vs + } + + podsConnectivityCheckResult := make(map[string]map[string]map[string]int) + for i := 0; i != s.VirtualNodesCount; i++ { + dp := s.createdNodeDPs[i] + vn := s.createdNodeVNs[i] + + var vsList []*appmeshv1beta1.VirtualService + for _, backend := range vn.Spec.Backends { + vsList = append(vsList, vsByName[backend.VirtualService.VirtualServiceName]) + } + ret := s.checkDeploymentVirtualServiceConnectivity(ctx, f, dp, vsList) + for k, v := range ret { + podsConnectivityCheckResult[k] = v + } + } + prettyJSON, err := json.MarshalIndent(podsConnectivityCheckResult, "", " ") + Expect(err).NotTo(HaveOccurred()) + utils.Logf("%v", string(prettyJSON)) +} + +func (s *DynamicStack) createMeshAndNamespace(ctx context.Context, f *framework.Framework) { + By("create a mesh", func() { + meshName := fmt.Sprintf("%s-%s", f.Options.ClusterName, utils.RandomDNS1123Label(6)) + mesh, err := f.K8sMeshClient.AppmeshV1beta1().Meshes().Create(&appmeshv1beta1.Mesh{ + ObjectMeta: metav1.ObjectMeta{ + Name: meshName, + }, + Spec: appmeshv1beta1.MeshSpec{}, + }) + Expect(err).NotTo(HaveOccurred()) + s.mesh = mesh + }) + + By(fmt.Sprintf("wait for mesh %s become active", s.mesh.Name), func() { + mesh, err := f.MeshManager.WaitUntilMeshActive(ctx, s.mesh) + Expect(err).NotTo(HaveOccurred()) + s.mesh = mesh + }) + + By("allocates a namespace", func() { + namespace, err := f.NSManager.AllocateNamespace(ctx, "appmesh") + Expect(err).NotTo(HaveOccurred()) + s.namespace = namespace + }) + + By("label namespace with appMesh inject", func() { + namespace := s.namespace.DeepCopy() + namespace.Labels = collection.MergeStringMap(s.namespace.Labels, map[string]string{ + "appmesh.k8s.aws/sidecarInjectorWebhook": "enabled", + }) + patch, err := k8s.CreateStrategicTwoWayMergePatch(s.namespace, namespace, corev1.Namespace{}) + Expect(err).NotTo(HaveOccurred()) + namespace, err = f.K8sClient.CoreV1().Namespaces().Patch(namespace.Name, types.StrategicMergePatchType, patch) + Expect(err).NotTo(HaveOccurred()) + s.namespace = namespace + }) +} + +func (s *DynamicStack) deleteMeshAndNamespace(ctx context.Context, f *framework.Framework) []error { + var deletionErrors []error + if s.namespace != nil { + By(fmt.Sprintf("delete namespace: %s", s.namespace.Name), func() { + foregroundDeletion := metav1.DeletePropagationForeground + if err := f.K8sClient.CoreV1().Namespaces().Delete(s.namespace.Name, &metav1.DeleteOptions{ + GracePeriodSeconds: aws.Int64(0), + PropagationPolicy: &foregroundDeletion, + }); err != nil { + utils.Logf("failed to delete namespace: %s due to %v", s.namespace.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + if s.mesh != nil { + By(fmt.Sprintf("delete mesh %s", s.mesh.Name), func() { + foregroundDeletion := metav1.DeletePropagationForeground + if err := f.K8sMeshClient.AppmeshV1beta1().Meshes().Delete(s.mesh.Name, &metav1.DeleteOptions{ + GracePeriodSeconds: aws.Int64(0), + PropagationPolicy: &foregroundDeletion, + }); err != nil { + utils.Logf("failed to delete mesh: %s due to %v", s.mesh.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + return deletionErrors +} + +func (s *DynamicStack) createCloudMapNamespace(ctx context.Context, f *framework.Framework) { + cmNamespace := fmt.Sprintf("%s-%s", f.Options.ClusterName, utils.RandomDNS1123Label(6)) + By(fmt.Sprintf("create cloudMap namespace %s", cmNamespace), func() { + resp, err := f.SDClient.CreatePrivateDnsNamespaceWithContext(ctx, &servicediscovery.CreatePrivateDnsNamespaceInput{ + Name: aws.String(cmNamespace), + Vpc: aws.String(f.Options.AWSVPCID), + }) + Expect(err).NotTo(HaveOccurred()) + s.cloudMapNamespace = cmNamespace + utils.Logf("created cloudMap namespace %s with operationID %s", cmNamespace, aws.StringValue(resp.OperationId)) + }) +} + +func (s *DynamicStack) deleteCloudMapNamespace(ctx context.Context, f *framework.Framework) []error { + var deletionErrors []error + if s.cloudMapNamespace != "" { + By(fmt.Sprintf("delete cloudMap namespace %s", s.cloudMapNamespace), func() { + var cmNamespaceID string + f.SDClient.ListNamespacesPagesWithContext(ctx, &servicediscovery.ListNamespacesInput{}, func(output *servicediscovery.ListNamespacesOutput, b bool) bool { + for _, ns := range output.Namespaces { + if aws.StringValue(ns.Name) == s.cloudMapNamespace { + cmNamespaceID = aws.StringValue(ns.Id) + return true + } + } + return false + }) + if cmNamespaceID == "" { + err := errors.Errorf("cannot find cloudMap namespace with name %s", s.cloudMapNamespace) + utils.Logf("failed to delete cloudMap namespace: %s due to %v", s.cloudMapNamespace, err) + deletionErrors = append(deletionErrors, err) + return + } + + // hummm, let's fix the controller bug in test cases first xD: + By(fmt.Sprintf("[bug workaround] clean up resources in cloudMap namespace %s", s.cloudMapNamespace), func() { + // give controller a break to deregister instance xD + time.Sleep(1 * time.Minute) + var cmServiceIDs []string + f.SDClient.ListServicesPagesWithContext(ctx, &servicediscovery.ListServicesInput{ + Filters: []*servicediscovery.ServiceFilter{ + { + Condition: aws.String(servicediscovery.FilterConditionEq), + Name: aws.String("NAMESPACE_ID"), + Values: aws.StringSlice([]string{cmNamespaceID}), + }, + }, + }, func(output *servicediscovery.ListServicesOutput, b bool) bool { + for _, svc := range output.Services { + cmServiceIDs = append(cmServiceIDs, aws.StringValue(svc.Id)) + } + return false + }) + for _, cmServiceID := range cmServiceIDs { + var cmInstanceIDs []string + f.SDClient.ListInstancesPagesWithContext(ctx, &servicediscovery.ListInstancesInput{ + ServiceId: aws.String(cmServiceID), + }, func(output *servicediscovery.ListInstancesOutput, b bool) bool { + for _, ins := range output.Instances { + cmInstanceIDs = append(cmInstanceIDs, aws.StringValue(ins.Id)) + } + return false + }) + + for _, cmInstanceID := range cmInstanceIDs { + if _, err := f.SDClient.DeregisterInstanceWithContext(ctx, &servicediscovery.DeregisterInstanceInput{ + ServiceId: aws.String(cmServiceID), + InstanceId: aws.String(cmInstanceID), + }); err != nil { + utils.Logf("failed to deregister instance due to %v", err) + deletionErrors = append(deletionErrors, err) + } + } + time.Sleep(30 * time.Second) + + if _, err := f.SDClient.DeleteServiceWithContext(ctx, &servicediscovery.DeleteServiceInput{ + Id: aws.String(cmServiceID), + }); err != nil { + utils.Logf("failed to deregister instance due to %v", err) + deletionErrors = append(deletionErrors, err) + } + } + }) + time.Sleep(30 * time.Second) + if _, err := f.SDClient.DeleteNamespaceWithContext(ctx, &servicediscovery.DeleteNamespaceInput{ + Id: aws.String(cmNamespaceID), + }); err != nil { + utils.Logf("failed to delete cloudMap namespace: %s due to %v", s.cloudMapNamespace, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + return deletionErrors +} + +func (s *DynamicStack) createResourcesForNodes(ctx context.Context, f *framework.Framework, mb *shared.ManifestBuilder) { + throttle := time.Tick(appMeshOperationRate) + + By("create all resources for nodes", func() { + s.createdNodeVNs = make([]*appmeshv1beta1.VirtualNode, s.VirtualNodesCount) + s.createdNodeDPs = make([]*appsv1.Deployment, s.VirtualNodesCount) + s.createdNodeSVCs = make([]*corev1.Service, s.VirtualNodesCount) + + var err error + + for i := 0; i != s.VirtualNodesCount; i++ { + instanceName := fmt.Sprintf("node-%d", i) + By(fmt.Sprintf("create VirtualNode for node #%d", i), func() { + vn := mb.BuildNodeVirtualNode(instanceName, nil) + + <-throttle + vn, err = f.K8sMeshClient.AppmeshV1beta1().VirtualNodes(s.namespace.Name).Create(vn) + Expect(err).NotTo(HaveOccurred()) + s.createdNodeVNs[i] = vn + }) + + By(fmt.Sprintf("create Deployment for node #%d", i), func() { + dp := mb.BuildNodeDeployment(instanceName, s.ReplicasPerVirtualNode) + dp, err = f.K8sClient.AppsV1().Deployments(s.namespace.Name).Create(dp) + Expect(err).NotTo(HaveOccurred()) + s.createdNodeDPs[i] = dp + }) + + By(fmt.Sprintf("create Service for node #%d", i), func() { + svc := mb.BuildNodeService(instanceName) + svc, err := f.K8sClient.CoreV1().Services(s.namespace.Name).Create(svc) + Expect(err).NotTo(HaveOccurred()) + s.createdNodeSVCs[i] = svc + }) + } + + By("wait all VirtualNodes become active", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i := 0; i != s.VirtualNodesCount; i++ { + wg.Add(1) + go func(nodeIndex int) { + defer wg.Done() + dp, err := f.VNManager.WaitUntilVirtualNodeActive(ctx, s.createdNodeVNs[nodeIndex]) + if err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + s.createdNodeVNs[nodeIndex] = dp + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all VirtualNodes become active due to errors: %v", waitErrors) + } + }) + + By("wait all deployments become ready", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i := 0; i != s.VirtualNodesCount; i++ { + wg.Add(1) + go func(nodeIndex int) { + defer wg.Done() + dp, err := f.DPManager.WaitUntilDeploymentReady(ctx, s.createdNodeDPs[nodeIndex]) + if err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + s.createdNodeDPs[nodeIndex] = dp + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all deployments become ready due to errors: %v", waitErrors) + } + }) + }) +} + +func (s *DynamicStack) deleteResourcesForNodes(ctx context.Context, f *framework.Framework) []error { + throttle := time.Tick(appMeshOperationRate) + + var deletionErrors []error + By("delete all resources for nodes", func() { + for i, svc := range s.createdNodeSVCs { + if svc == nil { + continue + } + By(fmt.Sprintf("delete Service for node #%d", i), func() { + if err := f.K8sClient.CoreV1().Services(svc.Namespace).Delete(svc.Name, &metav1.DeleteOptions{}); err != nil { + utils.Logf("failed to delete Service: %s/%s due to %v", svc.Namespace, svc.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + for i, dp := range s.createdNodeDPs { + if dp == nil { + continue + } + By(fmt.Sprintf("delete Deployment for node #%d", i), func() { + foregroundDeletion := metav1.DeletePropagationForeground + if err := f.K8sClient.AppsV1().Deployments(dp.Namespace).Delete(dp.Name, &metav1.DeleteOptions{ + GracePeriodSeconds: aws.Int64(0), + PropagationPolicy: &foregroundDeletion, + }); err != nil { + utils.Logf("failed to delete Deployment: %s/%s due to %v", dp.Namespace, dp.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + for i, vn := range s.createdNodeVNs { + if vn == nil { + continue + } + By(fmt.Sprintf("delete VirtualNode for node #%d", i), func() { + <-throttle + if err := f.K8sMeshClient.AppmeshV1beta1().VirtualNodes(vn.Namespace).Delete(vn.Name, &metav1.DeleteOptions{}); err != nil { + utils.Logf("failed to delete VirtualNode: %s/%s due to %v", vn.Namespace, vn.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + + By("wait all deployments become deleted", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i, dp := range s.createdNodeDPs { + if dp == nil { + continue + } + wg.Add(1) + go func(nodeIndex int) { + defer wg.Done() + if err := f.DPManager.WaitUntilDeploymentDeleted(ctx, s.createdNodeDPs[nodeIndex]); err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all deployments become deleted due to errors: %v", waitErrors) + } + }) + + By("wait all VirtualNodes become deleted", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i, vn := range s.createdNodeVNs { + if vn == nil { + continue + } + wg.Add(1) + go func(nodeIndex int) { + defer wg.Done() + if err := f.VNManager.WaitUntilVirtualNodeDeleted(ctx, s.createdNodeVNs[nodeIndex]); err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all VirtualNodes become active due to errors: %v", waitErrors) + } + }) + }) + return deletionErrors +} + +func (s *DynamicStack) createResourcesForServices(ctx context.Context, f *framework.Framework, mb *shared.ManifestBuilder) { + throttle := time.Tick(appMeshOperationRate) + + By("create all resources for services", func() { + s.createdServiceVSs = make([]*appmeshv1beta1.VirtualService, s.VirtualServicesCount) + s.createdServiceSVCs = make([]*corev1.Service, s.VirtualServicesCount) + + var err error + nextVirtualNodeIndex := 0 + for i := 0; i != s.VirtualServicesCount; i++ { + instanceName := fmt.Sprintf("service-%d", i) + By(fmt.Sprintf("create VirtualService for service #%d", i), func() { + var routeCfgs []shared.RouteToWeightedVirtualNodes + for routeIndex := 0; routeIndex != s.RoutesCountPerVirtualRouter; routeIndex++ { + var weightedTargets []shared.WeightedVirtualNode + for targetIndex := 0; targetIndex != s.TargetsCountPerRoute; targetIndex++ { + weightedTargets = append(weightedTargets, shared.WeightedVirtualNode{ + VirtualNodeName: s.createdNodeVNs[nextVirtualNodeIndex].Name, + Weight: 1, + }) + nextVirtualNodeIndex = (nextVirtualNodeIndex + 1) % s.VirtualNodesCount + } + routeCfgs = append(routeCfgs, shared.RouteToWeightedVirtualNodes{ + Path: fmt.Sprintf("/path-%d", routeIndex), + WeightedTargets: weightedTargets, + }) + } + vs := mb.BuildServiceVirtualService(instanceName, routeCfgs) + <-throttle + vs, err := f.K8sMeshClient.AppmeshV1beta1().VirtualServices(s.namespace.Name).Create(vs) + Expect(err).NotTo(HaveOccurred()) + s.createdServiceVSs[i] = vs + }) + + By(fmt.Sprintf("create Service for service #%d", i), func() { + svc := mb.BuildServiceService(instanceName) + svc, err = f.K8sClient.CoreV1().Services(s.namespace.Name).Create(svc) + Expect(err).NotTo(HaveOccurred()) + s.createdServiceSVCs[i] = svc + }) + } + + By("wait all VirtualService become active", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i := 0; i != s.VirtualServicesCount; i++ { + wg.Add(1) + go func(serviceIndex int) { + defer wg.Done() + vs, err := f.VSManager.WaitUntilVirtualServiceActive(ctx, s.createdServiceVSs[serviceIndex]) + if err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + s.createdServiceVSs[serviceIndex] = vs + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all VirtualService become active due to errors: %v", waitErrors) + } + }) + }) +} + +func (s *DynamicStack) deleteResourcesForServices(ctx context.Context, f *framework.Framework) []error { + throttle := time.Tick(appMeshOperationRate) + + var deletionErrors []error + By("delete all resources for services", func() { + for i, svc := range s.createdServiceSVCs { + if svc == nil { + continue + } + + By(fmt.Sprintf("delete Service for service #%d", i), func() { + if err := f.K8sClient.CoreV1().Services(svc.Namespace).Delete(svc.Name, &metav1.DeleteOptions{}); err != nil { + utils.Logf("failed to delete Service: %s/%s due to %v", svc.Namespace, svc.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + for i, vs := range s.createdServiceVSs { + if vs == nil { + continue + } + By(fmt.Sprintf("delete VirtualService for service #%d", i), func() { + <-throttle + if err := f.K8sMeshClient.AppmeshV1beta1().VirtualServices(vs.Namespace).Delete(vs.Name, &metav1.DeleteOptions{}); err != nil { + utils.Logf("failed to delete VirtualService: %s/%s due to %v", vs.Namespace, vs.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + + By("wait all VirtualService become deleted", func() { + var waitErrors []string + waitErrorsMutex := &sync.Mutex{} + var wg sync.WaitGroup + for i, vs := range s.createdServiceVSs { + if vs == nil { + continue + } + wg.Add(1) + go func(serviceIndex int) { + defer wg.Done() + if err := f.VSManager.WaitUntilVirtualServiceDeleted(ctx, s.createdServiceVSs[serviceIndex]); err != nil { + waitErrorsMutex.Lock() + waitErrors = append(waitErrors, err.Error()) + waitErrorsMutex.Unlock() + return + } + }(i) + } + wg.Wait() + if len(waitErrors) != 0 { + utils.Failf("failed to wait all VirtualService become deleted due to errors: %v", waitErrors) + } + }) + }) + return deletionErrors +} + +func (s *DynamicStack) grantVirtualNodesBackendAccess(ctx context.Context, f *framework.Framework) { + throttle := time.Tick(appMeshOperationRate) + + By("granting VirtualNodes backend access", func() { + nextVirtualServiceIndex := 0 + for i, vn := range s.createdNodeVNs { + if vn == nil { + continue + } + By(fmt.Sprintf("granting VirtualNode backend access for node #%d", i), func() { + var vnBackends []appmeshv1beta1.Backend + for backendIndex := 0; backendIndex != s.BackendsCountPerVirtualNode; backendIndex++ { + vnBackends = append(vnBackends, appmeshv1beta1.Backend{ + VirtualService: appmeshv1beta1.VirtualServiceBackend{ + VirtualServiceName: s.createdServiceVSs[nextVirtualServiceIndex].Name, + }, + }) + nextVirtualServiceIndex = (nextVirtualServiceIndex + 1) % s.VirtualServicesCount + } + + vnNew := vn.DeepCopy() + vnNew.Spec.Backends = vnBackends + patch, err := k8s.CreateJSONMergePatch(vn, vnNew, appmeshv1beta1.VirtualNode{}) + Expect(err).NotTo(HaveOccurred()) + <-throttle + vnNew, err = f.K8sMeshClient.AppmeshV1beta1().VirtualNodes(vnNew.Namespace).Patch(vnNew.Name, types.MergePatchType, patch) + Expect(err).NotTo(HaveOccurred()) + s.createdNodeVNs[i] = vnNew + }) + } + }) +} + +func (s *DynamicStack) revokeVirtualNodeBackendAccess(ctx context.Context, f *framework.Framework) []error { + throttle := time.Tick(appMeshOperationRate) + + var deletionErrors []error + By("revoking VirtualNodes backend access", func() { + for i, vn := range s.createdNodeVNs { + if vn == nil || len(vn.Spec.Backends) == 0 { + continue + } + By(fmt.Sprintf("revoking VirtualNode backend access for node #%d", i), func() { + vnNew := vn.DeepCopy() + vnNew.Spec.Backends = nil + patch, err := k8s.CreateJSONMergePatch(vn, vnNew, appmeshv1beta1.VirtualNode{}) + if err != nil { + utils.Logf("failed to revoke VirtualNode backend access : %s/%s due to %v", vn.Namespace, vn.Name, err) + deletionErrors = append(deletionErrors, err) + return + } + <-throttle + vnNew, err = f.K8sMeshClient.AppmeshV1beta1().VirtualNodes(vnNew.Namespace).Patch(vnNew.Name, types.MergePatchType, patch) + if err != nil { + utils.Logf("failed to revoke VirtualNode backend access : %s/%s due to %v", vn.Namespace, vn.Name, err) + deletionErrors = append(deletionErrors, err) + } + }) + } + }) + return deletionErrors +} + +func (s *DynamicStack) checkDeploymentVirtualServiceConnectivity(ctx context.Context, f *framework.Framework, + dp *appsv1.Deployment, vsList []*appmeshv1beta1.VirtualService) map[string]map[string]map[string]int { + sel := labels.Set(dp.Spec.Selector.MatchLabels) + opts := metav1.ListOptions{LabelSelector: sel.AsSelector().String()} + pods, err := f.K8sClient.CoreV1().Pods(dp.Namespace).List(opts) + Expect(err).NotTo(HaveOccurred()) + Expect(len(pods.Items)).NotTo(BeZero()) + + podsConnectivityCheckResult := make(map[string]map[string]map[string]int) + for i := range pods.Items { + pod := pods.Items[i].DeepCopy() + By(fmt.Sprintf("check pod %s/%s connectivity to services", pod.Namespace, pod.Name), func() { + podConnectivityCheckResult := s.checkPodVirtualServiceConnectivity(ctx, f, pod, vsList) + podsConnectivityCheckResult[fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)] = podConnectivityCheckResult + }) + } + return podsConnectivityCheckResult +} + +func (s *DynamicStack) checkPodVirtualServiceConnectivity(ctx context.Context, f *framework.Framework, + pod *corev1.Pod, vsList []*appmeshv1beta1.VirtualService) map[string]map[string]int { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var urls []string + for _, vs := range vsList { + for _, route := range vs.Spec.Routes { + path := route.Http.Match.Prefix + urls = append(urls, fmt.Sprintf("http://%s:%d%s", vs.Name, shared.AppContainerPort, path)) + } + } + + readyChan := make(chan struct{}) + portForwarder, err := k8s.NewPortForwarder(ctx, f.RestCfg, pod, []string{fmt.Sprintf("%d", shared.HttpProxyContainerPort)}, readyChan) + Expect(err).NotTo(HaveOccurred()) + + errChan := make(chan error) + go func(errChan chan error) { + errChan <- portForwarder.ForwardPorts() + }(errChan) + + proxyUrl, err := url.Parse(fmt.Sprintf("http://localhost:%d", shared.HttpProxyContainerPort)) + Expect(err).NotTo(HaveOccurred()) + proxyClient := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} + + type urlCheckResult struct { + url string + statusCode int + body string + err string + } + + urlCheckResultChan := make(chan urlCheckResult) + go func() { + var wg sync.WaitGroup + throttle := time.Tick(connectivityCheckRate) + for _, url := range urls { + for i := 0; i != s.ConnectivityCheckPerURL; i++ { + <-throttle + wg.Add(1) + go func(url string) { + defer wg.Done() + <-readyChan + + ret := urlCheckResult{ + url: url, + } + resp, err := proxyClient.Get(url) + if err != nil { + ret.err = err.Error() + urlCheckResultChan <- ret + return + } + ret.statusCode = resp.StatusCode + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + ret.err = err.Error() + urlCheckResultChan <- ret + return + } + ret.body = string(body) + urlCheckResultChan <- ret + }(url) + } + } + wg.Wait() + close(urlCheckResultChan) + }() + + urlRespCount := make(map[string]map[string]int) + for ret := range urlCheckResultChan { + if _, ok := urlRespCount[ret.url]; !ok { + urlRespCount[ret.url] = make(map[string]int) + } + payload := fmt.Sprintf("code: %v, body: %v, err: %v", ret.statusCode, ret.body, ret.err) + urlRespCount[ret.url][payload] += 1 + } + cancel() + Expect(<-errChan).To(BeNil()) + return urlRespCount +} diff --git a/test/e2e/fishapp/dynamic_stack_test.go b/test/e2e/fishapp/dynamic_stack_test.go new file mode 100644 index 000000000..ee3770c9c --- /dev/null +++ b/test/e2e/fishapp/dynamic_stack_test.go @@ -0,0 +1,66 @@ +package fishapp_test + +import ( + "context" + "fmt" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/fishapp" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/fishapp/shared" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework" + . "github.com/onsi/ginkgo" +) + +var _ = Describe("tests by dynamic generated stack", func() { + var ( + ctx context.Context + f *framework.Framework + dynamicStack *fishapp.DynamicStack + ) + + JustBeforeEach(func() { + ctx = context.Background() + f = framework.New(framework.GlobalOptions) + dynamicStack.Deploy(ctx, f) + }) + JustAfterEach(func() { + dynamicStack.Cleanup(ctx, f) + }) + + Context("normal test dimensions with DNS", func() { + BeforeEach(func() { + dynamicStack = &fishapp.DynamicStack{ + ServiceDiscoveryType: shared.DNSServiceDiscovery, + VirtualServicesCount: 5, + VirtualNodesCount: 5, + RoutesCountPerVirtualRouter: 2, + TargetsCountPerRoute: 4, + BackendsCountPerVirtualNode: 2, + ReplicasPerVirtualNode: 3, + ConnectivityCheckPerURL: 100, + } + }) + It("should behaves correctly", func() { + dynamicStack.ExpectBehavesCorrectly(ctx, f) + }) + }) + + Context("normal test dimensions with cloudMap", func() { + BeforeEach(func() { + dynamicStack = &fishapp.DynamicStack{ + ServiceDiscoveryType: shared.CloudMapServiceDiscovery, + VirtualServicesCount: 5, + VirtualNodesCount: 5, + RoutesCountPerVirtualRouter: 2, + TargetsCountPerRoute: 4, + BackendsCountPerVirtualNode: 2, + ReplicasPerVirtualNode: 3, + ConnectivityCheckPerURL: 100, + } + }) + It("should behaves correctly", func() { + dynamicStack.ExpectBehavesCorrectly(ctx, f) + var ans int + fmt.Println("press any thing to continue cleanup") + _, _ = fmt.Scan(&ans) + }) + }) +}) diff --git a/test/e2e/fishapp/fishapp_suite_test.go b/test/e2e/fishapp/fishapp_suite_test.go new file mode 100644 index 000000000..3a705f984 --- /dev/null +++ b/test/e2e/fishapp/fishapp_suite_test.go @@ -0,0 +1,13 @@ +package fishapp_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestFishApp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "FishApp Suite") +} diff --git a/test/e2e/fishapp/shared/manifest_builder.go b/test/e2e/fishapp/shared/manifest_builder.go new file mode 100644 index 000000000..2b7b60ce7 --- /dev/null +++ b/test/e2e/fishapp/shared/manifest_builder.go @@ -0,0 +1,275 @@ +package shared + +import ( + "fmt" + appmeshv1beta1 "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/apis/appmesh/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + defaultAppImage = "970805265562.dkr.ecr.us-west-2.amazonaws.com/colorteller:latest" + defaultHTTPProxyImage = "abhinavsingh/proxy.py:latest" +) + +const ( + AppContainerPort = 9080 + HttpProxyContainerPort = 8899 +) + +type ServiceDiscoveryType int + +const ( + DNSServiceDiscovery ServiceDiscoveryType = iota + CloudMapServiceDiscovery +) + +type ManifestBuilder struct { + MeshName string + Namespace string + ServiceDiscoveryType ServiceDiscoveryType + + // required when serviceDiscoveryType == CloudMapServiceDiscovery + CloudMapNamespace string +} + +func (b *ManifestBuilder) BuildNodeDeployment(instanceName string, replicas int32) *appsv1.Deployment { + labels := b.buildNodeSelectors(instanceName) + dpName := instanceName + dp := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: dpName, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: map[string]string{ + "appmesh.k8s.aws/mesh": b.MeshName, + "appmesh.k8s.aws/ports": fmt.Sprintf("%d", AppContainerPort), + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "app", + Image: defaultAppImage, + Ports: []corev1.ContainerPort{ + { + ContainerPort: AppContainerPort, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "SERVER_PORT", + Value: fmt.Sprintf("%d", AppContainerPort), + }, + { + Name: "COLOR", + Value: instanceName, + }, + }, + }, + { + Name: "http-proxy", + Image: defaultHTTPProxyImage, + Ports: []corev1.ContainerPort{ + { + ContainerPort: HttpProxyContainerPort, + }, + }, + Args: []string{ + "--hostname=0.0.0.0", + fmt.Sprintf("--port=%d", HttpProxyContainerPort), + }, + }, + }, + }, + }, + }, + } + return dp +} + +func (b *ManifestBuilder) BuildNodeService(instanceName string) *corev1.Service { + labels := b.buildNodeSelectors(instanceName) + svcName := b.buildNodeServiceName(instanceName) + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: svcName, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: labels, + Ports: []corev1.ServicePort{ + { + Port: AppContainerPort, + TargetPort: intstr.FromInt(AppContainerPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + return svc +} + +func (b *ManifestBuilder) BuildNodeVirtualNode(instanceName string, backendVirtualServices []string) *appmeshv1beta1.VirtualNode { + vnName := instanceName + var sd *appmeshv1beta1.ServiceDiscovery + switch b.ServiceDiscoveryType { + case DNSServiceDiscovery: + sd = b.buildNodeDNSServiceDiscovery(instanceName) + case CloudMapServiceDiscovery: + sd = b.buildNodeCloudMapServiceDiscovery(instanceName) + } + var backends []appmeshv1beta1.Backend + for _, backendVS := range backendVirtualServices { + backends = append(backends, appmeshv1beta1.Backend{ + VirtualService: appmeshv1beta1.VirtualServiceBackend{ + VirtualServiceName: backendVS, + }, + }) + } + vn := &appmeshv1beta1.VirtualNode{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: vnName, + }, + Spec: appmeshv1beta1.VirtualNodeSpec{ + MeshName: b.MeshName, + Listeners: []appmeshv1beta1.Listener{ + { + PortMapping: appmeshv1beta1.PortMapping{ + Port: AppContainerPort, + Protocol: "http", + }, + }, + }, + ServiceDiscovery: sd, + Backends: backends, + }, + } + return vn +} + +type RouteToWeightedVirtualNodes struct { + Path string + WeightedTargets []WeightedVirtualNode +} + +// WeightedVirtualNode is virtual node with weight +type WeightedVirtualNode struct { + VirtualNodeName string + Weight int64 +} + +func (b *ManifestBuilder) BuildServiceService(instanceName string) *corev1.Service { + svcName := instanceName + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: svcName, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + Port: AppContainerPort, + TargetPort: intstr.FromInt(AppContainerPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + return svc +} + +func (b *ManifestBuilder) BuildServiceVirtualService(instanceName string, routeCfgs []RouteToWeightedVirtualNodes) *appmeshv1beta1.VirtualService { + svcName := b.buildServiceServiceName(instanceName) + svcDNS := fmt.Sprintf("%s.%s", svcName, b.Namespace) + var routes []appmeshv1beta1.Route + for index, routeCfg := range routeCfgs { + var targets []appmeshv1beta1.WeightedTarget + for _, weightedTarget := range routeCfg.WeightedTargets { + targets = append(targets, appmeshv1beta1.WeightedTarget{ + VirtualNodeName: weightedTarget.VirtualNodeName, + Weight: weightedTarget.Weight, + }) + } + routes = append(routes, appmeshv1beta1.Route{ + Name: fmt.Sprintf("path-%d", index), + Http: &appmeshv1beta1.HttpRoute{ + Match: appmeshv1beta1.HttpRouteMatch{ + Prefix: routeCfg.Path, + }, + Action: appmeshv1beta1.HttpRouteAction{ + WeightedTargets: targets, + }, + }, + }) + } + vs := &appmeshv1beta1.VirtualService{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: svcDNS, + }, + Spec: appmeshv1beta1.VirtualServiceSpec{ + MeshName: b.MeshName, + VirtualRouter: &appmeshv1beta1.VirtualRouter{ + Listeners: []appmeshv1beta1.VirtualRouterListener{ + { + PortMapping: appmeshv1beta1.PortMapping{ + Port: AppContainerPort, + Protocol: "http", + }, + }, + }, + }, + Routes: routes, + }, + } + return vs +} + +func (b *ManifestBuilder) buildNodeDNSServiceDiscovery(instanceName string) *appmeshv1beta1.ServiceDiscovery { + nodeServiceName := b.buildNodeServiceName(instanceName) + nodeServiceDNS := fmt.Sprintf("%s.%s", nodeServiceName, b.Namespace) + return &appmeshv1beta1.ServiceDiscovery{ + Dns: &appmeshv1beta1.DnsServiceDiscovery{ + HostName: nodeServiceDNS, + }, + } +} + +func (b *ManifestBuilder) buildNodeCloudMapServiceDiscovery(instanceName string) *appmeshv1beta1.ServiceDiscovery { + nodeServiceName := b.buildNodeServiceName(instanceName) + return &appmeshv1beta1.ServiceDiscovery{ + CloudMap: &appmeshv1beta1.CloudMapServiceDiscovery{ + NamespaceName: b.CloudMapNamespace, + ServiceName: nodeServiceName, + }, + } +} + +func (b *ManifestBuilder) buildNodeSelectors(instanceName string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "fish-app", + "app.kubernetes.io/instance": instanceName, + } +} + +func (b *ManifestBuilder) buildNodeServiceName(instanceName string) string { + // I like to be explicit about implicit connections between Objects + return instanceName +} + +func (b *ManifestBuilder) buildServiceServiceName(instanceName string) string { + // I like to be explicit about implicit connections between Objects + return instanceName +} diff --git a/test/e2e/framework/collection/map.go b/test/e2e/framework/collection/map.go new file mode 100644 index 000000000..d5cc22e6f --- /dev/null +++ b/test/e2e/framework/collection/map.go @@ -0,0 +1,11 @@ +package collection + +func MergeStringMap(maps ...map[string]string) map[string]string { + ret := make(map[string]string) + for _, _map := range maps { + for k, v := range _map { + ret[k] = v + } + } + return ret +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go new file mode 100644 index 000000000..0e2423c36 --- /dev/null +++ b/test/e2e/framework/framework.go @@ -0,0 +1,75 @@ +package framework + +import ( + meshclientset "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/client/clientset/versioned" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/resource/deployment" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/resource/mesh" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/resource/namespace" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/resource/virtualnode" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/resource/virtualservice" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/servicediscovery" + "github.com/aws/aws-sdk-go/service/servicediscovery/servicediscoveryiface" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +type Framework struct { + Options Options + RestCfg *rest.Config + K8sClient kubernetes.Interface + K8sMeshClient meshclientset.Interface + + NSManager namespace.Manager + DPManager deployment.Manager + MeshManager mesh.Manager + VNManager virtualnode.Manager + VSManager virtualservice.Manager + + SDClient servicediscoveryiface.ServiceDiscoveryAPI +} + +func New(options Options) *Framework { + err := options.Validate() + Expect(err).NotTo(HaveOccurred()) + + restCfg, err := buildRestConfig(options) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err := kubernetes.NewForConfig(restCfg) + Expect(err).NotTo(HaveOccurred()) + + k8sMeshClient, err := meshclientset.NewForConfig(restCfg) + Expect(err).NotTo(HaveOccurred()) + + sess := session.Must(session.NewSession(aws.NewConfig().WithRegion(options.AWSRegion))) + sdClient := servicediscovery.New(sess) + f := &Framework{ + Options: options, + RestCfg: restCfg, + K8sClient: k8sClient, + K8sMeshClient: k8sMeshClient, + + NSManager: namespace.NewManager(k8sClient), + DPManager: deployment.NewManager(k8sClient), + MeshManager: mesh.NewManager(k8sMeshClient), + VNManager: virtualnode.NewManager(k8sMeshClient), + VSManager: virtualservice.NewManager(k8sMeshClient), + + SDClient: sdClient, + } + return f +} + +func buildRestConfig(options Options) (*rest.Config, error) { + restCfg, err := clientcmd.BuildConfigFromFlags("", options.KubeConfig) + if err != nil { + return nil, err + } + restCfg.QPS = 20 + restCfg.Burst = 50 + return restCfg, nil +} diff --git a/test/e2e/framework/k8s/patch.go b/test/e2e/framework/k8s/patch.go new file mode 100644 index 000000000..44e463816 --- /dev/null +++ b/test/e2e/framework/k8s/patch.go @@ -0,0 +1,37 @@ +package k8s + +import ( + "encoding/json" + jsonpatch "github.com/evanphx/json-patch" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/strategicpatch" +) + +func CreateStrategicTwoWayMergePatch(oldObj metav1.Object, newObj metav1.Object, objType interface{}) ([]byte, error) { + oldData, err := json.Marshal(oldObj) + if err != nil { + return nil, errors.Wrapf(err, "failed to Marshal oldObj: %s/%s", oldObj.GetName(), oldObj.GetName()) + } + + newData, err := json.Marshal(newObj) + if err != nil { + return nil, errors.Wrapf(err, "failed to Marshal newObj: %s/%s", newObj.GetName(), newObj.GetName()) + } + + return strategicpatch.CreateTwoWayMergePatch(oldData, newData, objType) +} + +func CreateJSONMergePatch(oldObj metav1.Object, newObj metav1.Object, _ interface{}) ([]byte, error) { + oldData, err := json.Marshal(oldObj) + if err != nil { + return nil, errors.Wrapf(err, "failed to Marshal oldObj: %s/%s", oldObj.GetName(), oldObj.GetName()) + } + + newData, err := json.Marshal(newObj) + if err != nil { + return nil, errors.Wrapf(err, "failed to Marshal newObj: %s/%s", newObj.GetName(), newObj.GetName()) + } + + return jsonpatch.CreateMergePatch(oldData, newData) +} diff --git a/test/e2e/framework/k8s/port_forward.go b/test/e2e/framework/k8s/port_forward.go new file mode 100644 index 000000000..2b3bca834 --- /dev/null +++ b/test/e2e/framework/k8s/port_forward.go @@ -0,0 +1,33 @@ +package k8s + +import ( + "context" + "fmt" + "github.com/onsi/ginkgo" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" + "net/http" +) + +// NewPortForwarder return a new port forwarder +func NewPortForwarder(ctx context.Context, restCfg *rest.Config, pod *corev1.Pod, ports []string, readyChan chan struct{}) (*portforward.PortForwarder, error) { + cs, _ := kubernetes.NewForConfig(restCfg) + req := cs.RESTClient().Post(). + Resource("pods"). + Namespace(pod.Namespace). + Name(pod.Name). + SubResource("portforward") + url := req.URL() + // the generated url above is incorrect, fix them below + url.Path = fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", pod.Namespace, pod.Name) + + transport, upgrader, err := spdy.RoundTripperFor(restCfg) + if err != nil { + return nil, err + } + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, url) + return portforward.New(dialer, ports, ctx.Done(), readyChan, nil, ginkgo.GinkgoWriter) +} diff --git a/test/e2e/framework/options.go b/test/e2e/framework/options.go new file mode 100644 index 000000000..7efa26b57 --- /dev/null +++ b/test/e2e/framework/options.go @@ -0,0 +1,43 @@ +package framework + +import ( + "flag" + "github.com/pkg/errors" + "k8s.io/client-go/tools/clientcmd" +) + +var GlobalOptions Options + +func init() { + GlobalOptions.BindFlags() +} + +type Options struct { + KubeConfig string + ClusterName string + AWSRegion string + AWSVPCID string +} + +func (options *Options) BindFlags() { + flag.StringVar(&options.KubeConfig, clientcmd.RecommendedConfigPathFlag, "", "Path to kubeconfig containing embedded authinfo (required)") + flag.StringVar(&options.ClusterName, "cluster-name", "", `Kubernetes cluster name (required)`) + flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`) + flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`) +} + +func (options *Options) Validate() error { + if len(options.KubeConfig) == 0 { + return errors.Errorf("%s must be set!", clientcmd.RecommendedConfigPathFlag) + } + if len(options.ClusterName) == 0 { + return errors.Errorf("%s must be set!", "cluster-name") + } + if len(options.AWSRegion) == 0 { + return errors.Errorf("%s must be set!", "aws-region") + } + if len(options.AWSVPCID) == 0 { + return errors.Errorf("%s must be set!", "aws-vpc-id") + } + return nil +} diff --git a/test/e2e/framework/resource/deployment/manager.go b/test/e2e/framework/resource/deployment/manager.go new file mode 100644 index 000000000..51ea06468 --- /dev/null +++ b/test/e2e/framework/resource/deployment/manager.go @@ -0,0 +1,61 @@ +package deployment + +import ( + "context" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + appsv1 "k8s.io/api/apps/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +type Manager interface { + WaitUntilDeploymentReady(ctx context.Context, dp *appsv1.Deployment) (*appsv1.Deployment, error) + + WaitUntilDeploymentDeleted(ctx context.Context, dp *appsv1.Deployment) error +} + +func NewManager(cs kubernetes.Interface) Manager { + return &defaultManager{ + cs: cs, + } +} + +type defaultManager struct { + cs kubernetes.Interface +} + +func (m *defaultManager) WaitUntilDeploymentReady(ctx context.Context, dp *appsv1.Deployment) (*appsv1.Deployment, error) { + var ( + observedDP *appsv1.Deployment + err error + ) + return observedDP, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + observedDP, err = m.cs.AppsV1().Deployments(dp.Namespace).Get(dp.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + if observedDP.Status.UpdatedReplicas == (*dp.Spec.Replicas) && + observedDP.Status.Replicas == (*dp.Spec.Replicas) && + observedDP.Status.AvailableReplicas == (*dp.Spec.Replicas) && + observedDP.Status.ObservedGeneration >= dp.Generation { + return true, nil + } + return false, nil + }, ctx.Done()) +} + +func (m *defaultManager) WaitUntilDeploymentDeleted(ctx context.Context, dp *appsv1.Deployment) error { + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + if _, err := m.cs.AppsV1().Deployments(dp.Namespace).Get(dp.Name, metav1.GetOptions{}); err != nil { + if apierrs.IsNotFound(err) { + return true, nil + } + utils.Logf("Error while waiting for Deployment to be terminated: %v", err) + return false, nil + } + return false, nil + }, ctx.Done()) +} diff --git a/test/e2e/framework/resource/mesh/manager.go b/test/e2e/framework/resource/mesh/manager.go new file mode 100644 index 000000000..d5a87b179 --- /dev/null +++ b/test/e2e/framework/resource/mesh/manager.go @@ -0,0 +1,44 @@ +package mesh + +import ( + "context" + appmeshv1beta1 "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/apis/appmesh/v1beta1" + meshclientset "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/client/clientset/versioned" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +type Manager interface { + WaitUntilMeshActive(ctx context.Context, mesh *appmeshv1beta1.Mesh) (*appmeshv1beta1.Mesh, error) +} + +func NewManager(cs meshclientset.Interface) Manager { + return &defaultManager{cs: cs} +} + +type defaultManager struct { + cs meshclientset.Interface +} + +func (m *defaultManager) WaitUntilMeshActive(ctx context.Context, mesh *appmeshv1beta1.Mesh) (*appmeshv1beta1.Mesh, error) { + var ( + observedMesh *appmeshv1beta1.Mesh + err error + ) + return observedMesh, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + observedMesh, err = m.cs.AppmeshV1beta1().Meshes().Get(mesh.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + for _, condition := range observedMesh.Status.Conditions { + if condition.Type == appmeshv1beta1.MeshActive && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + + return false, nil + }, ctx.Done()) +} diff --git a/test/e2e/framework/resource/namespace/manager.go b/test/e2e/framework/resource/namespace/manager.go new file mode 100644 index 000000000..25acac0e8 --- /dev/null +++ b/test/e2e/framework/resource/namespace/manager.go @@ -0,0 +1,72 @@ +package namespace + +import ( + "context" + "fmt" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +type Manager interface { + AllocateNamespace(ctx context.Context, baseName string) (*corev1.Namespace, error) +} + +func NewManager(cs kubernetes.Interface) Manager { + return &defaultManager{ + cs: cs, + } +} + +type defaultManager struct { + cs kubernetes.Interface +} + +func (m *defaultManager) AllocateNamespace(ctx context.Context, baseName string) (*corev1.Namespace, error) { + name, err := m.findAvailableNamespaceName(ctx, baseName) + if err != nil { + return nil, err + } + namespaceObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + var namespace *corev1.Namespace + if err := wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + var err error + namespace, err = m.cs.CoreV1().Namespaces().Create(namespaceObj) + if err != nil { + utils.Logf("Unexpected error while creating namespace: %v", err) + return false, nil + } + return true, nil + }, ctx.Done()); err != nil { + return nil, err + } + return namespace, nil +} + +// findAvailableNamespaceName random namespace name starting with baseName. +func (m *defaultManager) findAvailableNamespaceName(ctx context.Context, baseName string) (string, error) { + var name string + err := wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + name = fmt.Sprintf("%v-%v", baseName, utils.RandomDNS1123Label(6)) + _, err := m.cs.CoreV1().Namespaces().Get(name, metav1.GetOptions{}) + if err == nil { + // Already taken + return false, nil + } + if apierrs.IsNotFound(err) { + return true, nil + } + utils.Logf("Unexpected error while getting namespace: %v", err) + return false, nil + }, ctx.Done()) + + return name, err +} diff --git a/test/e2e/framework/resource/virtualnode/manager.go b/test/e2e/framework/resource/virtualnode/manager.go new file mode 100644 index 000000000..6ed0b1446 --- /dev/null +++ b/test/e2e/framework/resource/virtualnode/manager.go @@ -0,0 +1,60 @@ +package virtualnode + +import ( + "context" + appmeshv1beta1 "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/apis/appmesh/v1beta1" + meshclientset "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/client/clientset/versioned" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +type Manager interface { + WaitUntilVirtualNodeActive(ctx context.Context, vn *appmeshv1beta1.VirtualNode) (*appmeshv1beta1.VirtualNode, error) + + WaitUntilVirtualNodeDeleted(ctx context.Context, vn *appmeshv1beta1.VirtualNode) error +} + +func NewManager(cs meshclientset.Interface) Manager { + return &defaultManager{cs: cs} +} + +type defaultManager struct { + cs meshclientset.Interface +} + +func (m *defaultManager) WaitUntilVirtualNodeActive(ctx context.Context, vn *appmeshv1beta1.VirtualNode) (*appmeshv1beta1.VirtualNode, error) { + var ( + observedVN *appmeshv1beta1.VirtualNode + err error + ) + return observedVN, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + observedVN, err = m.cs.AppmeshV1beta1().VirtualNodes(vn.Namespace).Get(vn.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + for _, condition := range observedVN.Status.Conditions { + if condition.Type == appmeshv1beta1.VirtualNodeActive && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + + return false, nil + }, ctx.Done()) +} + +func (m *defaultManager) WaitUntilVirtualNodeDeleted(ctx context.Context, vn *appmeshv1beta1.VirtualNode) error { + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + if _, err := m.cs.AppmeshV1beta1().VirtualNodes(vn.Namespace).Get(vn.Name, metav1.GetOptions{}); err != nil { + if apierrs.IsNotFound(err) { + return true, nil + } + utils.Logf("Error while waiting for VirtualNode to be terminated: %v", err) + return false, nil + } + return false, nil + }, ctx.Done()) +} diff --git a/test/e2e/framework/resource/virtualservice/manager.go b/test/e2e/framework/resource/virtualservice/manager.go new file mode 100644 index 000000000..018b8b298 --- /dev/null +++ b/test/e2e/framework/resource/virtualservice/manager.go @@ -0,0 +1,69 @@ +package virtualservice + +import ( + "context" + appmeshv1beta1 "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/apis/appmesh/v1beta1" + meshclientset "github.com/aws/aws-app-mesh-controller-for-k8s/pkg/client/clientset/versioned" + "github.com/aws/aws-app-mesh-controller-for-k8s/test/e2e/framework/utils" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +type Manager interface { + WaitUntilVirtualServiceActive(ctx context.Context, vs *appmeshv1beta1.VirtualService) (*appmeshv1beta1.VirtualService, error) + + WaitUntilVirtualServiceDeleted(ctx context.Context, vs *appmeshv1beta1.VirtualService) error +} + +func NewManager(cs meshclientset.Interface) Manager { + return &defaultManager{cs: cs} +} + +type defaultManager struct { + cs meshclientset.Interface +} + +func (m *defaultManager) WaitUntilVirtualServiceActive(ctx context.Context, vs *appmeshv1beta1.VirtualService) (*appmeshv1beta1.VirtualService, error) { + var ( + observedVS *appmeshv1beta1.VirtualService + err error + ) + return observedVS, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + observedVS, err = m.cs.AppmeshV1beta1().VirtualServices(vs.Namespace).Get(vs.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + conditions := make(map[appmeshv1beta1.VirtualServiceConditionType]corev1.ConditionStatus) + for _, condition := range observedVS.Status.Conditions { + conditions[condition.Type] = condition.Status + } + readyConditionTypes := []appmeshv1beta1.VirtualServiceConditionType{ + appmeshv1beta1.VirtualServiceActive, + appmeshv1beta1.VirtualRouterActive, + appmeshv1beta1.RoutesActive, + } + for _, conditionType := range readyConditionTypes { + status, ok := conditions[conditionType] + if !ok || status != corev1.ConditionTrue { + return false, nil + } + } + return true, nil + }, ctx.Done()) +} + +func (m *defaultManager) WaitUntilVirtualServiceDeleted(ctx context.Context, vs *appmeshv1beta1.VirtualService) error { + return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { + if _, err := m.cs.AppmeshV1beta1().VirtualServices(vs.Namespace).Get(vs.Name, metav1.GetOptions{}); err != nil { + if apierrs.IsNotFound(err) { + return true, nil + } + utils.Logf("Error while waiting for VirtualService to be terminated: %v", err) + return false, nil + } + return false, nil + }, ctx.Done()) +} diff --git a/test/e2e/framework/utils/log.go b/test/e2e/framework/utils/log.go new file mode 100644 index 000000000..0f9b8fdfa --- /dev/null +++ b/test/e2e/framework/utils/log.go @@ -0,0 +1,25 @@ +package utils + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo" +) + +func nowStamp() string { + return time.Now().Format(time.StampMilli) +} + +func log(level string, format string, args ...interface{}) { + fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...) +} + +func Logf(format string, args ...interface{}) { + log("INFO", format, args...) +} + +func Failf(format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + ginkgo.Fail(msg) +} diff --git a/test/e2e/framework/utils/name.go b/test/e2e/framework/utils/name.go new file mode 100644 index 000000000..ca6c0f06d --- /dev/null +++ b/test/e2e/framework/utils/name.go @@ -0,0 +1,18 @@ +package utils + +import ( + "crypto/rand" + "encoding/hex" + "io" +) + +// RandomDNS1123Label generates a random DNS1123 compatible label with specified length +func RandomDNS1123Label(length int) string { + seedLen := (length + 1) / 2 + seedBuf := make([]byte, seedLen) + io.ReadFull(rand.Reader, seedBuf[:]) + + labelBuf := make([]byte, seedLen*2) + hex.Encode(labelBuf, seedBuf) + return string(labelBuf[:length]) +} diff --git a/test/e2e/framework/utils/poll.go b/test/e2e/framework/utils/poll.go new file mode 100644 index 000000000..0ab189eed --- /dev/null +++ b/test/e2e/framework/utils/poll.go @@ -0,0 +1,8 @@ +package utils + +import "time" + +const ( + PollIntervalShort = 2 * time.Second + PollIntervalMedium = 10 * time.Second +)