Skip to content
This repository has been archived by the owner on Dec 4, 2024. It is now read-only.

Commit

Permalink
[release/3][BACKPORT] fix release branch tests (#1070)
Browse files Browse the repository at this point in the history 
* pep8 again

* add DISPATCH_BUILD_NAME to all test containers

* use github cli to determine the merge target

* fix bash linting

* fix makefile shell

* log in github cli

* bump github cli

* logging in fails if GITHUB_TOKEN is set

* set -x

* actually use the kba-branch flag

* workaround for bad "make clean" in kubeaddons-tests

* fix wrong module name

Co-authored-by: Joe Julian <[email protected]>
  • Loading branch information
@joejulian
mesosphere-mergebot and joejulian authored Apr 16, 2021
1 parent bc8b985 commit 0a4c247
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 97 deletions.
128 changes: 56 additions & 72 deletions Dispatchfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ load("github.com/mesosphere/cluster-claim-controller/starlark/claim@master", "ma
load("github.com/mesosphere/dispatch-catalog/starlark/stable/[email protected]", "dindTask")
load("github.com/mesosphere/dispatch-catalog/starlark/stable/[email protected]", "git_resource")
load("github.com/mesosphere/dispatch-catalog/starlark/stable/[email protected]", "secret_var")
load("github.com/mesosphere/dispatch-catalog/starlark/stable/[email protected]", "push", "pull_request", "cron", "clean", "task_step_result", "storage_resource")
load("github.com/mesosphere/dispatch-catalog/starlark/stable/[email protected]",
"push", "pull_request", "cron", "clean", "task_step_result", "storage_resource")
load("github.com/mesosphere/dispatch-tasks/bump_charts/bump_charts@master", "bump_charts")

git_clone_dirname = "kba-git-src"
Expand All @@ -28,14 +29,29 @@ kind_cluster = make_kind_cluster(kind_cluster_name, git_clone_dirname)
install_kind_cluster = make_kind_cluster(kind_install_cluster_name, git_clone_dirname)
upgrade_kind_cluster = make_kind_cluster(kind_upgrade_cluster_name, git_clone_dirname)

common_env = [
k8s.corev1.EnvVar(name="ADDON_TESTS_PER_ADDON_WAIT_DURATION", value="10m"),
k8s.corev1.EnvVar(name="ADDON_TESTS_SETUP_WAIT_DURATION", value="60m"),
k8s.corev1.EnvVar(name="AWS_ACCESS_KEY_ID", valueFrom=secret_var( "d2iq-dispatch-aws-creds", "AWS_ACCESS_KEY_ID")),
k8s.corev1.EnvVar(name="AWS_REGION", value="us-west-2"),
k8s.corev1.EnvVar(name="AWS_SECRET_ACCESS_KEY", valueFrom=secret_var( "d2iq-dispatch-aws-creds", "AWS_SECRET_ACCESS_KEY")),
k8s.corev1.EnvVar(name="DISPATCH_BUILD_NAME", value="$(context.build.name)"),
k8s.corev1.EnvVar(name="DISPATCH_CI", value="true"),
k8s.corev1.EnvVar(name="GIT_TERMINAL_PROMPT", value="1"),
k8s.corev1.EnvVar(name="GITHUB_TOKEN", valueFrom=secret_var("d2iq-dispatch-basic-auth", "password")),
k8s.corev1.EnvVar(name="GOPRIVATE", value="github.com/mesosphere"),
k8s.corev1.EnvVar(name="KONVOY_VERSION", value="v1.6.1"),
k8s.corev1.EnvVar(name="KUBEADDONS_ADDON_DELETE_DURATION", value="5m"),
k8s.corev1.EnvVar(name="SSH_KEY_BASE64", valueFrom=secret_var( "d2iq-dispatch-git-ssh-base64", "ssh-privatekey-base64")),
]

dindTask("lint-yaml",
inputs=[git_clone_dirname],
steps=[
k8s.corev1.Container(
name="fetch-master",
name="fetch-branches",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
args=["git", "fetch", "origin", "master"]
args=["git", "fetch", "origin"]
),

k8s.corev1.Container(
Expand All @@ -53,31 +69,20 @@ dindTask("test-install",
fetch_kubeconfig(kind_install_cluster_name, git_clone_dirname),

k8s.corev1.Container(
name="fetch-master",
name="fetch-branches",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
args=["git", "fetch", "origin", "master"]),
args=["git", "fetch", "origin"]),

k8s.corev1.Container(
name="test-install",
image="mesosphere/kubeaddons-ci:dispatch@sha256:80a6f3c8775133b4ad2fab3d44ab4fbb4b541dbc03fcdb296b13f67611427d8c",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
command=["/entrypoint.sh", "make", "dispatch-test-install-upgrade"],

env=[
k8s.corev1.EnvVar(name="DISPATCH_CI", value="true"),
k8s.corev1.EnvVar(name="AWS_REGION", value="us-west-2"),
k8s.corev1.EnvVar(name="AWS_ACCESS_KEY_ID", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_ACCESS_KEY_ID")),
k8s.corev1.EnvVar(name="AWS_SECRET_ACCESS_KEY", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_SECRET_ACCESS_KEY")),
k8s.corev1.EnvVar(name="SSH_KEY_BASE64", valueFrom=secret_var("d2iq-dispatch-git-ssh-base64", "ssh-privatekey-base64")),
k8s.corev1.EnvVar(name="ADDON_TESTS_PER_ADDON_WAIT_DURATION", value="10m"),
k8s.corev1.EnvVar(name="GIT_TERMINAL_PROMPT", value="1"),
k8s.corev1.EnvVar(name="ADDON_TESTS_SETUP_WAIT_DURATION", value="60m"),
k8s.corev1.EnvVar(name="KUBEADDONS_ADDON_DELETE_DURATION", value="5m"),
k8s.corev1.EnvVar(name="GOPRIVATE", value="github.com/mesosphere/kubeaddons,github.com/mesosphere/ksphere-testing-framework"),
k8s.corev1.EnvVar(name="KONVOY_VERSION", value="v1.6.1"),
env= common_env + [
k8s.corev1.EnvVar(name="CLAIM_NAME", value=task_step_result(kind_install_cluster_name, "cluster-claim")),
k8s.corev1.EnvVar(name="KBA_TESTGROUP_TYPE", value="install"),
k8s.corev1.EnvVar(name = "CLAIM_NAME", value = task_step_result(kind_install_cluster_name, "cluster-claim"))
],
],

resources=k8s.corev1.ResourceRequirements(
requests={
Expand All @@ -92,31 +97,20 @@ dindTask("test-upgrade",
fetch_kubeconfig(kind_upgrade_cluster_name, git_clone_dirname),

k8s.corev1.Container(
name="fetch-master",
name="fetch-branches",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
args=["git", "fetch", "origin", "master"]),
args=["git", "fetch", "origin"]),

k8s.corev1.Container(
name="test-upgrade",
image="mesosphere/kubeaddons-ci:dispatch@sha256:80a6f3c8775133b4ad2fab3d44ab4fbb4b541dbc03fcdb296b13f67611427d8c",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
command=["/entrypoint.sh", "make", "dispatch-test-install-upgrade"],

env=[
k8s.corev1.EnvVar(name="DISPATCH_CI", value="true"),
k8s.corev1.EnvVar(name="AWS_REGION", value="us-west-2"),
k8s.corev1.EnvVar(name="AWS_ACCESS_KEY_ID", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_ACCESS_KEY_ID")),
k8s.corev1.EnvVar(name="AWS_SECRET_ACCESS_KEY", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_SECRET_ACCESS_KEY")),
k8s.corev1.EnvVar(name="SSH_KEY_BASE64", valueFrom=secret_var("d2iq-dispatch-git-ssh-base64", "ssh-privatekey-base64")),
k8s.corev1.EnvVar(name="ADDON_TESTS_PER_ADDON_WAIT_DURATION", value="10m"),
k8s.corev1.EnvVar(name="GIT_TERMINAL_PROMPT", value="1"),
k8s.corev1.EnvVar(name="ADDON_TESTS_SETUP_WAIT_DURATION", value="60m"),
k8s.corev1.EnvVar(name="KUBEADDONS_ADDON_DELETE_DURATION", value="5m"),
k8s.corev1.EnvVar(name="GOPRIVATE", value="github.com/mesosphere/kubeaddons,github.com/mesosphere/ksphere-testing-framework"),
k8s.corev1.EnvVar(name="KONVOY_VERSION", value="v1.6.1"),
env= common_env + [
k8s.corev1.EnvVar(name="CLAIM_NAME", value=task_step_result(kind_upgrade_cluster_name, "cluster-claim")),
k8s.corev1.EnvVar(name="KBA_TESTGROUP_TYPE", value="upgrade"),
k8s.corev1.EnvVar(name = "CLAIM_NAME", value = task_step_result(kind_upgrade_cluster_name, "cluster-claim"))
],
],

resources=k8s.corev1.ResourceRequirements(
requests={
Expand All @@ -135,73 +129,63 @@ dindTask("test-aws-install-addons",
outputs=[konvoy_artifacts_resource_name],
steps=[
k8s.corev1.Container(
name="fetch-master",
name="fetch-branches",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
args=["git", "fetch", "origin", "master"]),
args=["git", "fetch", "origin"]),

k8s.corev1.Container(
name="test-addons",
image="mesosphere/kubeaddons-ci:dispatch@sha256:80a6f3c8775133b4ad2fab3d44ab4fbb4b541dbc03fcdb296b13f67611427d8c",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
command=["/entrypoint.sh", "make", "test-aws"],

env=[
k8s.corev1.EnvVar(name="ARTIFACTS_DIR", value="$(resources.outputs.{}.path)".format(konvoy_artifacts_resource_name)),
k8s.corev1.EnvVar(name="DISPATCH_CI", value="true"),
k8s.corev1.EnvVar(name="GOPRIVATE", value="github.com/mesosphere"),
k8s.corev1.EnvVar(name="AWS_REGION", value="us-west-2"),
env=common_env + [
k8s.corev1.EnvVar(name="ARTIFACTS_DIR", value="$(resources.outputs.{}.path)".format(
konvoy_artifacts_resource_name)),
k8s.corev1.EnvVar(name="KBA_TESTGROUP_TYPE", value="install"),
k8s.corev1.EnvVar(name="AWS_ACCESS_KEY_ID", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_ACCESS_KEY_ID")),
k8s.corev1.EnvVar(name="AWS_SECRET_ACCESS_KEY", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_SECRET_ACCESS_KEY")),
k8s.corev1.EnvVar(name="SSH_KEY_BASE64", valueFrom=secret_var("d2iq-dispatch-git-ssh-base64", "ssh-privatekey-base64"))
],
resources=k8s.corev1.ResourceRequirements(
requests={
"cpu": k8s.resource_quantity("1000m"),
"memory": k8s.resource_quantity("3Gi")},
limits={
"cpu": k8s.resource_quantity("8000m"),
"memory": k8s.resource_quantity("15Gi")}
)
)])
],
resources=k8s.corev1.ResourceRequirements(
requests={
"cpu": k8s.resource_quantity("1000m"),
"memory": k8s.resource_quantity("3Gi")},
limits={
"cpu": k8s.resource_quantity("8000m"),
"memory": k8s.resource_quantity("15Gi")}
)
)])

dindTask("test-aws-upgrade-addons",
inputs=[git_clone_dirname],
outputs=[konvoy_artifacts_upgrade_name],
steps=[
k8s.corev1.Container(
name="fetch-master",
name="fetch-branches",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
args=["git", "fetch", "origin", "master"]),
args=["git", "fetch", "origin"]),

k8s.corev1.Container(
name="test-addons",
image="mesosphere/kubeaddons-ci:dispatch@sha256:80a6f3c8775133b4ad2fab3d44ab4fbb4b541dbc03fcdb296b13f67611427d8c",
workingDir="/workspace/{git_clone_dirname}/".format(git_clone_dirname=git_clone_dirname),
command=["/entrypoint.sh", "make", "test-aws"],

env=[
k8s.corev1.EnvVar(name="DISPATCH_CI", value="true"),
k8s.corev1.EnvVar(name="GOPRIVATE", value="github.com/mesosphere"),
k8s.corev1.EnvVar(name="AWS_REGION", value="us-west-2"),
env=common_env + [
k8s.corev1.EnvVar(name="ARTIFACTS_DIR", value="$(resources.outputs.{}.path)".format(
konvoy_artifacts_upgrade_name)),
k8s.corev1.EnvVar(name="KBA_TESTGROUP_TYPE", value="upgrade"),
k8s.corev1.EnvVar(name="AWS_ACCESS_KEY_ID", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_ACCESS_KEY_ID")),
k8s.corev1.EnvVar(name="AWS_SECRET_ACCESS_KEY", valueFrom=secret_var("d2iq-dispatch-aws-creds", "AWS_SECRET_ACCESS_KEY")),
k8s.corev1.EnvVar(name="SSH_KEY_BASE64", valueFrom=secret_var("d2iq-dispatch-git-ssh-base64", "ssh-privatekey-base64")),
k8s.corev1.EnvVar(name="ARTIFACTS_DIR", value="$(resources.outputs.{}.path)".format(konvoy_artifacts_upgrade_name))
],
],

resources=k8s.corev1.ResourceRequirements(
requests={
"cpu": k8s.resource_quantity("1000m"),
"memory": k8s.resource_quantity("3Gi")
},
limits={
"cpu": k8s.resource_quantity("8000m"),
"memory": k8s.resource_quantity("15Gi")}
)
},
limits={
"cpu": k8s.resource_quantity("8000m"),
"memory": k8s.resource_quantity("15Gi")}
)

)])
)])

clean_install_kind = cleanup(install_kind_cluster, "test-install", git_clone_dirname)
clean_upgrade_kind = cleanup(upgrade_kind_cluster, "test-upgrade", git_clone_dirname)
Expand Down
15 changes: 9 additions & 6 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
MKFILE_DIR := $(dir $(MKFILE_PATH))
SHELL := /bin/bash -euo pipefail
SHELL := /bin/bash
SHELLFLAGS := -euo pipefail
comma := ,
empty :=
space := $(empty) $(empty)
Expand All @@ -10,7 +11,7 @@ commaspace := $(comma)$(empty)
# ------------------------------------------------------------------------------
# Configuration - Versions
# ------------------------------------------------------------------------------
GITHUB_CLI_VERSION := 1.5.0
GITHUB_CLI_VERSION := 1.8.1

# ------------------------------------------------------------------------------
# Configuration - Golang
Expand All @@ -32,7 +33,7 @@ export GO111MODULE := on
# ------------------------------------------------------------------------------
# Configuration - Binaries
# ------------------------------------------------------------------------------
GITHUB_CLI_BIN := $(MKFILE_DIR)/bin/linux/$(GOARCH)/gh-$(GITHUB_CLI_VERSION)
export GITHUB_CLI_BIN := $(MKFILE_DIR)/bin/linux/$(GOARCH)/gh-$(GITHUB_CLI_VERSION)
RELEASE_NOTES_TOOL_BIN := $(MKFILE_DIR)/bin/$(GOOS)/$(GOARCH)/release-notes

# ------------------------------------------------------------------------------
Expand Down Expand Up @@ -101,14 +102,16 @@ kind-test: kubeaddons-tests
.PHONY: clean
clean:
ifneq (,$(wildcard kubeaddons-tests/Makefile))
make -f kubeaddons-tests/Makefile clean KUBEADDONS_REPO=kubernetes-base-addons
-make -f kubeaddons-tests/Makefile clean KUBEADDONS_REPO=kubernetes-base-addons
endif
-rm -rf kubeaddons-tests
-rm kba-kubeconfig-*

.PHONY: dispatch-test-install-upgrade
dispatch-test-install-upgrade:
pushd test; ./dispatch_test_install_upgrade.sh $(KBA_BRANCH); popd;
.ONESHELL:
dispatch-test-install-upgrade: $(GITHUB_CLI_BIN)
cd test
./dispatch_test_install_upgrade.sh

.PHONY: test-aws
test-aws: test/konvoy
Expand Down
69 changes: 66 additions & 3 deletions test/addons_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ const (
patchStorageClass = `{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}`

comRepoURL = "https://github.com/mesosphere/kubeaddons-community"
comRepoRef = "master"
comRepoRemote = "origin"
comRepoRef = "master"

defaultKBARepoRef = "master"

Expand Down Expand Up @@ -228,6 +228,69 @@ func TestAzureGroup(t *testing.T) {
// Private Functions
// -----------------------------------------------------------------------------

func checkIfUpgradeIsNeeded(t *testing.T, groupname string) (bool, []v1beta2.AddonInterface, error) {
t.Logf("determining which addons in group %s need to be upgrade tested", groupname)

var doUpgrade bool
addons := groups[groupname]
for _, addon := range addons {
if err := overrides(addon); err != nil {
return false, nil, err
}
}

addonDeploymentsArray := make([]v1beta2.AddonInterface, 0)
for _, newAddon := range addons {
t.Logf("verifying whether upgrade testing is needed for addon %s", newAddon.GetName())
oldAddon, err := testutils.GetLatestAddonRevisionFromLocalRepoBranch("../", comRepoRemote, kbaRepoRef, newAddon.GetName())
if err != nil {
if strings.Contains(err.Error(), "directory not found") {
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), kbaRepoRef)
addonDeploymentsArray = append(addonDeploymentsArray, newAddon)
continue
}
return false, nil, err
}
if oldAddon == nil {
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), kbaRepoRef)
addonDeploymentsArray = append(addonDeploymentsArray, newAddon)
continue // new addon, upgrade test not needed
}

// Apply overrides to oldAddon to ensure it is deployed with the necessary value overrides
if err := overrides(oldAddon); err != nil {
return false, nil, err
}

t.Logf("determining old and new versions for upgrade testing addon %s", newAddon.GetName())
oldRev := oldAddon.GetAnnotations()[constants.AddonRevisionAnnotation]
oldVersion, err := semver.Parse(strings.TrimPrefix(oldRev, "v"))
if err != nil {
return false, nil, err
}
newRev := newAddon.GetAnnotations()[constants.AddonRevisionAnnotation]
newVersion, err := semver.Parse(strings.TrimPrefix(newRev, "v"))
if err != nil {
return false, nil, err
}

if newVersion.EQ(oldVersion) {
t.Logf("skipping upgrade test for addon %s, it has not been updated", newAddon.GetName())
addonDeploymentsArray = append(addonDeploymentsArray, oldAddon)
continue
} else if oldVersion.GT(newVersion) {
return false, nil, fmt.Errorf("revisions for addon %s are broken, previous revision %s is newer than current %s", newAddon.GetName(), oldVersion, newVersion)
}

t.Logf("found old version of addon %s %s (revision %s) and new version %s (revision %s)", newAddon.GetName(), oldRev, oldVersion, newVersion, newRev)
// for upgraded addons, add the oldAddon (running previous version) to deployments
addonDeploymentsArray = append(addonDeploymentsArray, oldAddon)
doUpgrade = true
}

return doUpgrade, addonDeploymentsArray, nil
}

func createNodeVolumes(numberVolumes int, nodePrefix string, node *v1alpha4.Node) error {
dockerClient, err := docker.NewClientWithOpts(docker.FromEnv)
if err != nil {
Expand Down Expand Up @@ -358,13 +421,13 @@ func testgroup(t *testing.T, groupname string, version string, jobs ...clusterTe
oldAddon, err := testutils.GetLatestAddonRevisionFromLocalRepoBranch("../", comRepoRemote, kbaRepoRef, newAddon.GetName())
if err != nil {
if strings.Contains(err.Error(), "directory not found") {
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), comRepoRef)
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), kbaRepoRef)
continue
}
return err
}
if oldAddon == nil {
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), comRepoRef)
t.Logf("no need to upgrade test %s, it appears to be a new addon (no previous revisions found in branch %s)", newAddon.GetName(), kbaRepoRef)
continue // new addon, upgrade test not needed
}

Expand Down
Loading

0 comments on commit 0a4c247

Please sign in to comment.