From 68d2044b12843b6ba9141bc0c260354ac2b098fa Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Sat, 30 Jan 2021 17:11:30 -0600 Subject: [PATCH 1/2] kubetest2 - add terraform support --- tests/e2e/kubetest2-kops/deployer/common.go | 8 ++ tests/e2e/kubetest2-kops/deployer/create.go | 9 ++ tests/e2e/kubetest2-kops/deployer/deployer.go | 4 + tests/e2e/kubetest2-kops/deployer/down.go | 6 + tests/e2e/pkg/target/terraform.go | 117 ++++++++++++++++++ tests/e2e/pkg/util/zip.go | 76 ++++++++++++ 6 files changed, 220 insertions(+) create mode 100644 tests/e2e/pkg/target/terraform.go create mode 100644 tests/e2e/pkg/util/zip.go diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index ed04dea64b93f..f5bc1a2686296 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog/v2" "k8s.io/kops/tests/e2e/pkg/kops" + "k8s.io/kops/tests/e2e/pkg/target" ) func (d *deployer) init() error { @@ -71,6 +72,13 @@ func (d *deployer) initialize() error { if d.SSHUser == "" { d.SSHUser = os.Getenv("KUBE_SSH_USER") } + if d.TerraformVersion != "" { + t, err := target.NewTerraform(d.TerraformVersion) + if err != nil { + return err + } + d.terraform = t + } return nil } diff --git a/tests/e2e/kubetest2-kops/deployer/create.go b/tests/e2e/kubetest2-kops/deployer/create.go index 8107543c2c49a..1ad7573295f3c 100644 --- a/tests/e2e/kubetest2-kops/deployer/create.go +++ b/tests/e2e/kubetest2-kops/deployer/create.go @@ -46,6 +46,10 @@ func (d *deployer) replace() error { "--admin", "--name", d.ClusterName, } + if d.terraform != nil { + args = append(args, "--target", "terraform", "--out", d.terraform.Dir()) + } + klog.Info(strings.Join(args, " ")) cmd = exec.Command(args[0], args[1:]...) @@ -56,5 +60,10 @@ func (d *deployer) replace() error { if err != nil { return err } + if d.terraform != nil { + if err := d.terraform.InitApply(); err != nil { + return err + } + } return nil } diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index 5ec019085cd4c..aa8867cc93550 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -25,6 +25,7 @@ import ( "github.com/spf13/pflag" "k8s.io/klog/v2" "k8s.io/kops/tests/e2e/kubetest2-kops/builder" + "k8s.io/kops/tests/e2e/pkg/target" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -59,6 +60,8 @@ type deployer struct { SSHPublicKeyPath string `flag:"ssh-public-key" desc:"The path to the public key passed to the cloud provider"` SSHUser string `flag:"ssh-user" desc:"The SSH user to use for SSH access to instances"` + TerraformVersion string `flag:"terraform-version" desc:"The version of terraform to use for applying the cluster"` + ArtifactsDir string `flag:"-"` AdminAccess string `flag:"admin-access" desc:"The CIDR to restrict kubernetes API access"` @@ -67,6 +70,7 @@ type deployer struct { // manifestPath is the location of the rendered manifest based on TemplatePath manifestPath string + terraform *target.Terraform } // assert that New implements types.NewDeployer diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go index 88ead53e0c18f..f0476e42957ce 100644 --- a/tests/e2e/kubetest2-kops/deployer/down.go +++ b/tests/e2e/kubetest2-kops/deployer/down.go @@ -31,6 +31,12 @@ func (d *deployer) Down() error { klog.Warningf("Dumping cluster logs at the start of Down() failed: %s", err) } + if d.terraform != nil { + if err := d.terraform.Destroy(); err != nil { + return err + } + } + args := []string{ d.KopsBinaryPath, "delete", "cluster", "--name", d.ClusterName, diff --git a/tests/e2e/pkg/target/terraform.go b/tests/e2e/pkg/target/terraform.go new file mode 100644 index 0000000000000..3eddcd78c7c0e --- /dev/null +++ b/tests/e2e/pkg/target/terraform.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package target + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "runtime" + "strings" + + "k8s.io/klog/v2" + "k8s.io/kops/tests/e2e/pkg/util" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +// Terraform represents a set of terraform commands to be ran against a directory +// containing a kops cluster's .tf output +type Terraform struct { + dir string + terraformPath string +} + +// NewTerraform creates a new Terraform object +func NewTerraform(version string) (*Terraform, error) { + var b bytes.Buffer + url := fmt.Sprintf("https://releases.hashicorp.com/terraform/%v/terraform_%v_%v_%v.zip", version, version, runtime.GOOS, runtime.GOARCH) + + if err := util.HTTPGETWithHeaders(url, nil, &b); err != nil { + return nil, err + } + binaryDir, err := util.UnzipToTempDir(b.Bytes()) + if err != nil { + return nil, err + } + tfDir, err := ioutil.TempDir("", "kops-terraform") + if err != nil { + return nil, err + } + t := &Terraform{ + dir: tfDir, + terraformPath: filepath.Join(binaryDir, "terraform"), + } + return t, nil +} + +// Dir returns the directory containing the kops-generated .tf files +func (t *Terraform) Dir() string { + return t.dir +} + +// InitApply runs `terraform init` and `terraform apply` in the specified directory +func (t *Terraform) InitApply() error { + args := []string{ + t.terraformPath, "init", + } + klog.Info(strings.Join(args, " ")) + + cmd := exec.Command(args[0], args[1:]...) + cmd.SetDir(t.dir) + + exec.InheritOutput(cmd) + err := cmd.Run() + if err != nil { + return err + } + + args = []string{ + t.terraformPath, "apply", + "-auto-approve", + } + klog.Info(strings.Join(args, " ")) + + cmd = exec.Command(args[0], args[1:]...) + cmd.SetDir(t.dir) + + exec.InheritOutput(cmd) + err = cmd.Run() + if err != nil { + return err + } + return nil +} + +// Destroy runs `terraform destroy` in the specified directory +func (t *Terraform) Destroy() error { + args := []string{ + t.terraformPath, "destroy", + "-auto-approve", + } + klog.Info(strings.Join(args, " ")) + + cmd := exec.Command(args[0], args[1:]...) + cmd.SetDir(t.dir) + + exec.InheritOutput(cmd) + err := cmd.Run() + if err != nil { + return err + } + return nil +} diff --git a/tests/e2e/pkg/util/zip.go b/tests/e2e/pkg/util/zip.go new file mode 100644 index 0000000000000..5957ef49782c8 --- /dev/null +++ b/tests/e2e/pkg/util/zip.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// UnzipToTempDir will decompress the provided bytes into a temporary directory that is returned +func UnzipToTempDir(data []byte) (string, error) { + reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", err + } + dir, err := ioutil.TempDir("", "") + if err != nil { + return "", err + } + + for _, r := range reader.File { + fileNamePath := filepath.Join(dir, r.Name) + if !strings.HasPrefix(fileNamePath, filepath.Clean(dir)+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid file path: %v", fileNamePath) + } + + if r.FileInfo().IsDir() { + os.MkdirAll(fileNamePath, os.ModePerm) + continue + } + + if err = os.MkdirAll(filepath.Dir(fileNamePath), os.ModePerm); err != nil { + return "", err + } + + output, err := os.OpenFile(fileNamePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, r.Mode()) + if err != nil { + return "", err + } + + fileReader, err := r.Open() + if err != nil { + return "", err + } + + _, err = io.Copy(output, fileReader) + + output.Close() + fileReader.Close() + + if err != nil { + return "", err + } + } + return dir, nil +} From 6e8a329f7d6cf959be0cdedebbe2ba810f50f739 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Mon, 15 Feb 2021 23:02:15 -0600 Subject: [PATCH 2/2] Validate cluster 10 times to mimic kubetest1 and ensure nodes readiness is not flapping --- tests/e2e/kubetest2-kops/deployer/up.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go index 994c502d78360..9144dbbb365ae 100644 --- a/tests/e2e/kubetest2-kops/deployer/up.go +++ b/tests/e2e/kubetest2-kops/deployer/up.go @@ -134,6 +134,7 @@ func (d *deployer) IsUp() (bool, error) { args := []string{ d.KopsBinaryPath, "validate", "cluster", "--name", d.ClusterName, + "--count", "10", "--wait", "15m", } klog.Info(strings.Join(args, " "))