Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use github.com/gofrs/flock to lock handler #731

Merged
merged 3 commits into from
Apr 27, 2021
Merged

Use github.com/gofrs/flock to lock handler #731

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f7f6a2
Use github.com/gofrs/flock to lock handler
qinqon Apr 26, 2021
1ad9ff7
Replace handler readiness probe
qinqon Apr 23, 2021
5caab4c
Test nmstate handler lock mechanism
qinqon Apr 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions test/e2e/daemonset/daemonset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ func GetEventually(daemonSetKey types.NamespacedName) AsyncAssertion {
}, 180*time.Second, 1*time.Second)
}

func GetConsistently(daemonSetKey types.NamespacedName) AsyncAssertion {
return Consistently(func() (appsv1.DaemonSet, error) {
daemonSet := appsv1.DaemonSet{}
err := testenv.Client.Get(context.TODO(), daemonSetKey, &daemonSet)
return daemonSet, err
}, 15*time.Second, 1*time.Second)
}

// GetDaemonSetList returns a DaemonSetList matching the labels passed
func GetList(filteringLabels map[string]string) (appsv1.DaemonSetList, error) {
ds := appsv1.DaemonSetList{}
Expand Down
77 changes: 52 additions & 25 deletions test/e2e/operator/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package operator

import (
"context"
"fmt"
"os"
"testing"
"time"

Expand All @@ -11,44 +13,39 @@ import (
ginkgoreporters "kubevirt.io/qe-tools/pkg/ginkgo-reporters"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
dynclient "sigs.k8s.io/controller-runtime/pkg/client"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

nmstatev1beta1 "github.com/nmstate/kubernetes-nmstate/api/v1beta1"
testenv "github.com/nmstate/kubernetes-nmstate/test/env"
knmstatereporter "github.com/nmstate/kubernetes-nmstate/test/reporter"
)

var (
t *testing.T
nodes []string
startTime time.Time
t *testing.T
nodes []string
startTime time.Time
defaultNMState = nmstatev1beta1.NMState{
ObjectMeta: metav1.ObjectMeta{
Name: "nmstate",
Namespace: "nmstate",
},
}
webhookKey = types.NamespacedName{Namespace: "nmstate", Name: "nmstate-webhook"}
handlerKey = types.NamespacedName{Namespace: "nmstate", Name: "nmstate-handler"}
handlerLabels = map[string]string{"component": "kubernetes-nmstate-handler"}
)

var _ = BeforeSuite(func() {

logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

testenv.Start()

})

func TestMain(m *testing.M) {
func TestE2E(t *testing.T) {
testenv.TestMain()
}

func TestE2E(tapi *testing.T) {
t = tapi
RegisterFailHandler(Fail)

By("Getting node list from cluster")
nodeList := corev1.NodeList{}
err := testenv.Client.List(context.TODO(), &nodeList, &dynclient.ListOptions{})
Expect(err).ToNot(HaveOccurred())
for _, node := range nodeList.Items {
nodes = append(nodes, node.Name)
}

reporters := make([]Reporter, 0)
reporters = append(reporters, knmstatereporter.New("test_logs/e2e/operator", testenv.OperatorNamespace, nodes))
if ginkgoreporters.Polarion.Run {
Expand All @@ -61,8 +58,38 @@ func TestE2E(tapi *testing.T) {
RunSpecsWithDefaultAndCustomReporters(t, "Operator E2E Test Suite", reporters)
}

var _ = BeforeEach(func() {
var _ = BeforeSuite(func() {

// Change to root directory some test expect that
os.Chdir("../../../")

logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))

testenv.Start()

By("Getting node list from cluster")
nodeList := corev1.NodeList{}
err := testenv.Client.List(context.TODO(), &nodeList, &dynclient.ListOptions{})
Expect(err).ToNot(HaveOccurred())
for _, node := range nodeList.Items {
nodes = append(nodes, node.Name)
}
})

var _ = AfterEach(func() {
var _ = AfterSuite(func() {
uninstallNMState(defaultNMState)
})

func installNMState(nmstate nmstatev1beta1.NMState) {
By(fmt.Sprintf("Creating NMState CR '%s'", nmstate.Name))
err := testenv.Client.Create(context.TODO(), &nmstate)
ExpectWithOffset(1, err).ToNot(HaveOccurred(), "NMState CR created without error")
}

func uninstallNMState(nmstate nmstatev1beta1.NMState) {
By(fmt.Sprintf("Deleting NMState CR '%s'", nmstate.Name))
err := testenv.Client.Delete(context.TODO(), &nmstate, &client.DeleteOptions{})
if !apierrors.IsNotFound(err) {
Expect(err).ToNot(HaveOccurred(), "NMState CR successfully removed")
}
}
82 changes: 51 additions & 31 deletions test/e2e/operator/nmstate_install_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,35 @@ import (

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

nmstatev1beta1 "github.com/nmstate/kubernetes-nmstate/api/v1beta1"
"github.com/nmstate/kubernetes-nmstate/test/e2e/daemonset"
"github.com/nmstate/kubernetes-nmstate/test/e2e/deployment"

"github.com/nmstate/kubernetes-nmstate/test/cmd"
testenv "github.com/nmstate/kubernetes-nmstate/test/env"
)

var (
defaultNMState = nmstatev1beta1.NMState{
ObjectMeta: metav1.ObjectMeta{
Name: "nmstate",
Namespace: "nmstate",
},
}
webhookKey = types.NamespacedName{Namespace: "nmstate", Name: "nmstate-webhook"}
handlerKey = types.NamespacedName{Namespace: "nmstate", Name: "nmstate-handler"}
handlerLabels = map[string]string{"component": "kubernetes-nmstate-handler"}
)

var _ = Describe("NMState operator", func() {
Context("when installed for the first time", func() {
BeforeEach(func() {
installDefaultNMState()
installNMState(defaultNMState)
})
It("should deploy daemonset and webhook deployment", func() {
daemonset.GetEventually(handlerKey).Should(daemonset.BeReady())
deployment.GetEventually(webhookKey).Should(deployment.BeReady())
})
AfterEach(func() {
uninstallDefaultNMState()
uninstallNMState(defaultNMState)
})
})
Context("when NMState is installed", func() {
It("should list one NMState CR", func() {
installDefaultNMState()
installNMState(defaultNMState)
daemonset.GetEventually(handlerKey).Should(daemonset.BeReady())
ds, err := daemonset.GetList(handlerLabels)
Expect(err).ToNot(HaveOccurred(), "List daemon sets in namespace nmstate succeeds")
Expand All @@ -71,7 +60,7 @@ var _ = Describe("NMState operator", func() {
})
Context("and uninstalled", func() {
BeforeEach(func() {
uninstallDefaultNMState()
uninstallNMState(defaultNMState)
})
It("should uninstall handler and webhook", func() {
Eventually(func() bool {
Expand All @@ -85,24 +74,55 @@ var _ = Describe("NMState operator", func() {
})
})
})
Context("when another handler is installed with different namespace", func() {
var (
operatorNamespace = "nmstate-alt"
)
BeforeEach(func() {
installNMState(defaultNMState)
daemonset.GetEventually(handlerKey).Should(daemonset.BeReady())
installOperator(operatorNamespace)
})
AfterEach(func() {
uninstallNMState(defaultNMState)
uninstallOperator(operatorNamespace)
installOperator("nmstate")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is the operator installed in "nmstate" namespace at the end?
If it's done to test that operator can be installed and uninstalled at one namespace and then installed in another, wouldn't it be better to have another spec for it?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test-e2e-operator test suite spec the operator to be installed, but the test we are adding here remove the nmstate operator to check that the handler installed from different namespace is blocked until the "nmstate" one is delete.

The install here ensure that after that test the nmstate operator is there as expected at the other test cases.

})
It("should wait on the old one to be deleted", func() {
By("Checking handler is locked")
daemonset.GetConsistently(types.NamespacedName{Namespace: operatorNamespace, Name: "nmstate-handler"}).ShouldNot(daemonset.BeReady())
uninstallOperator("nmstate")
By("Checking handler is unlocked after deleting old one")
daemonset.GetEventually(types.NamespacedName{Namespace: operatorNamespace, Name: "nmstate-handler"}).Should(daemonset.BeReady())
})
})
})

func installNMState(nmstate nmstatev1beta1.NMState) {
err := testenv.Client.Create(context.TODO(), &nmstate)
Expect(err).ToNot(HaveOccurred(), "NMState CR created without error")
}

func installDefaultNMState() {
installNMState(defaultNMState)
}
func installOperator(namespace string) error {
By(fmt.Sprintf("Creating NMState operator with namespace '%s'", namespace))
_, err := cmd.Run("make", false, fmt.Sprintf("OPERATOR_NAMESPACE=%s", namespace), fmt.Sprintf("HANDLER_NAMESPACE=%s", namespace), "IMAGE_REGISTRY=registry:5000", "manifests")
Expect(err).ToNot(HaveOccurred())

func uninstallNMState(nmstate nmstatev1beta1.NMState) {
err := testenv.Client.Delete(context.TODO(), &nmstate, &client.DeleteOptions{})
if !apierrors.IsNotFound(err) {
Expect(err).ToNot(HaveOccurred(), "NMState CR successfully removed")
manifestsDir := "build/_output/manifests/"
manifests := []string{"namespace.yaml", "service_account.yaml", "operator.yaml", "role.yaml", "role_binding.yaml"}
for _, manifest := range manifests {
_, err = cmd.Kubectl("apply", "-f", manifestsDir+manifest)
Expect(err).ToNot(HaveOccurred())
}
deployment.GetEventually(types.NamespacedName{Namespace: namespace, Name: "nmstate-operator"}).Should(deployment.BeReady())

return nil
}

func uninstallDefaultNMState() {
uninstallNMState(defaultNMState)
func uninstallOperator(namespace string) {
By(fmt.Sprintf("Deleting namespace '%s'", namespace))
ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
},
}
Expect(testenv.Client.Delete(context.TODO(), &ns)).To(SatisfyAny(Succeed(), WithTransform(apierrors.IsNotFound, BeTrue())))
Eventually(func() error {
return testenv.Client.Get(context.TODO(), types.NamespacedName{Name: namespace}, &ns)
}, 2*time.Minute, 5*time.Second).Should(SatisfyAll(HaveOccurred(), WithTransform(apierrors.IsNotFound, BeTrue())))
}