Skip to content

Commit

Permalink
add CNINode integration tests (#479)
Browse files Browse the repository at this point in the history
* add CNINode integration tests

* address PR comments

* updating log statements

* add retry in VerifyCNINode
  • Loading branch information
sushrk authored Oct 13, 2024
1 parent 3e74da5 commit 21961a7
Show file tree
Hide file tree
Showing 14 changed files with 338 additions and 73 deletions.
3 changes: 1 addition & 2 deletions pkg/utils/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -237,7 +236,7 @@ func GetSourceAcctAndArn(roleARN, region, clusterName string) (string, string, s

// PodHasENIRequest will return true if first container of pod spec has request for eni indicating
// it needs trunk interface from vpc-rc
func PodHasENIRequest(pod *v1.Pod) bool {
func PodHasENIRequest(pod *corev1.Pod) bool {
if pod == nil {
return false
}
Expand Down
12 changes: 12 additions & 0 deletions pkg/utils/set.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@

package utils

import (
"github.com/aws/aws-sdk-go/service/ec2"
)

// Difference returns a-b, elements present in a and not in b
func Difference[T comparable](a, b []T) (diff []T) {
m := make(map[T]struct{})
Expand All @@ -35,3 +39,11 @@ func GetKeyValSlice(m map[string]string) (key []string, val []string) {
}
return
}

func GetTagKeyValueMap(tagSet []*ec2.Tag) map[string]string {
m := make(map[string]string)
for _, tag := range tagSet {
m[*tag.Key] = *tag.Value
}
return m
}
2 changes: 1 addition & 1 deletion scripts/test/run-integration-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ function run_integration_tests(){
echo "skipping Windows tests"
fi
(cd $INTEGRATION_TEST_DIR/webhook && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=5m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
# (cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail
(cd $INTEGRATION_TEST_DIR/cninode && CGO_ENABLED=0 ginkgo --skip=LOCAL $EXTRA_GINKGO_FLAGS -v -timeout=10m -- -cluster-kubeconfig=$KUBE_CONFIG_PATH -cluster-name=$CLUSTER_NAME --aws-region=$REGION --aws-vpc-id $VPC_ID) || TEST_RESULT=fail

if [[ "$TEST_RESULT" == fail ]]; then
exit 1
Expand Down
63 changes: 33 additions & 30 deletions test/framework/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
eniConfig "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
sgp "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1beta1"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/autoscaling"
ec2Manager "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/aws/ec2"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/configmap"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/controller"
Expand All @@ -42,21 +43,22 @@ import (
)

type Framework struct {
Options Options
K8sClient client.Client
ec2Client *ec2.EC2
DeploymentManager deployment.Manager
PodManager pod.Manager
EC2Manager *ec2Manager.Manager
SAManager serviceaccount.Manager
NSManager namespace.Manager
SGPManager *sgpManager.Manager
SVCManager service.Manager
JobManager jobs.Manager
NodeManager node.Manager
ControllerManager controller.Manager
RBACManager rbac.Manager
ConfigMapManager configmap.Manager
Options Options
K8sClient client.Client
ec2Client *ec2.EC2
DeploymentManager deployment.Manager
PodManager pod.Manager
EC2Manager *ec2Manager.Manager
SAManager serviceaccount.Manager
NSManager namespace.Manager
SGPManager *sgpManager.Manager
SVCManager service.Manager
JobManager jobs.Manager
NodeManager node.Manager
ControllerManager controller.Manager
RBACManager rbac.Manager
ConfigMapManager configmap.Manager
AutoScalingManager autoscaling.Manager
}

func New(options Options) *Framework {
Expand Down Expand Up @@ -91,20 +93,21 @@ func New(options Options) *Framework {
ec2 := ec2.New(sess, &aws.Config{Region: aws.String(options.AWSRegion)})

return &Framework{
K8sClient: k8sClient,
ec2Client: ec2,
PodManager: pod.NewManager(k8sClient, k8sSchema, config),
DeploymentManager: deployment.NewManager(k8sClient),
EC2Manager: ec2Manager.NewManager(ec2, options.AWSVPCID),
SAManager: serviceaccount.NewManager(k8sClient, config),
NSManager: namespace.NewManager(k8sClient),
SGPManager: sgpManager.NewManager(k8sClient),
SVCManager: service.NewManager(k8sClient),
JobManager: jobs.NewManager(k8sClient),
NodeManager: node.NewManager(k8sClient),
ControllerManager: controller.NewManager(k8sClient),
RBACManager: rbac.NewManager(k8sClient),
ConfigMapManager: configmap.NewManager(k8sClient),
Options: options,
K8sClient: k8sClient,
ec2Client: ec2,
PodManager: pod.NewManager(k8sClient, k8sSchema, config),
DeploymentManager: deployment.NewManager(k8sClient),
EC2Manager: ec2Manager.NewManager(ec2, options.AWSVPCID),
SAManager: serviceaccount.NewManager(k8sClient, config),
NSManager: namespace.NewManager(k8sClient),
SGPManager: sgpManager.NewManager(k8sClient),
SVCManager: service.NewManager(k8sClient),
JobManager: jobs.NewManager(k8sClient),
NodeManager: node.NewManager(k8sClient),
ControllerManager: controller.NewManager(k8sClient),
RBACManager: rbac.NewManager(k8sClient),
ConfigMapManager: configmap.NewManager(k8sClient),
AutoScalingManager: autoscaling.NewManager(sess),
Options: options,
}
}
64 changes: 64 additions & 0 deletions test/framework/resource/aws/autoscaling/manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package autoscaling

import (
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/autoscaling"
"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
)

type Manager interface {
DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error)
UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error
}

type defaultManager struct {
autoscalingiface.AutoScalingAPI
}

func NewManager(session *session.Session) Manager {
return &defaultManager{
AutoScalingAPI: autoscaling.New(session),
}
}

func (d defaultManager) DescribeAutoScalingGroup(autoScalingGroupName string) ([]*autoscaling.Group, error) {
describeAutoScalingGroupIp := &autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: aws.StringSlice([]string{autoScalingGroupName}),
}
asg, err := d.AutoScalingAPI.DescribeAutoScalingGroups(describeAutoScalingGroupIp)
if err != nil {
return nil, err
}
if len(asg.AutoScalingGroups) == 0 {
return nil, fmt.Errorf("failed to find asg %s", autoScalingGroupName)
}

return asg.AutoScalingGroups, nil
}

func (d defaultManager) UpdateAutoScalingGroup(asgName string, desiredSize, minSize, maxSize int64) error {
updateASGInput := &autoscaling.UpdateAutoScalingGroupInput{
AutoScalingGroupName: aws.String(asgName),
DesiredCapacity: aws.Int64(desiredSize),
MaxSize: aws.Int64(maxSize),
MinSize: aws.Int64(minSize),
}
_, err := d.AutoScalingAPI.UpdateAutoScalingGroup(updateASGInput)
return err
}
10 changes: 10 additions & 0 deletions test/framework/resource/k8s/node/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package node

import (
"context"
"strings"

cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
Expand All @@ -32,6 +33,7 @@ type Manager interface {
GetNodeList() (*v1.NodeList, error)
GetCNINode(node *v1.Node) (*cninode.CNINode, error)
GetCNINodeList() (*cninode.CNINodeList, error)
GetInstanceID(node *v1.Node) string
}

type defaultManager struct {
Expand Down Expand Up @@ -117,3 +119,11 @@ func (d *defaultManager) GetNodeList() (*v1.NodeList, error) {
err := d.k8sClient.List(context.TODO(), list)
return list, err
}

func (d *defaultManager) GetInstanceID(node *v1.Node) string {
if node.Spec.ProviderID != "" {
id := strings.Split(node.Spec.ProviderID, "/")
return id[len(id)-1]
}
return ""
}
67 changes: 54 additions & 13 deletions test/framework/resource/k8s/node/wrapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,71 @@ package node

import (
"context"
"fmt"

cninode "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
)

func GetNodeAndWaitTillCapacityPresent(manager Manager, ctx context.Context, os string, expectedResource string) *v1.NodeList {

func GetNodeAndWaitTillCapacityPresent(manager Manager, os string, expectedResource string) *v1.NodeList {
observedNodeList := &v1.NodeList{}
var err error
err = wait.Poll(utils.PollIntervalShort, utils.ResourceCreationTimeout, func() (bool, error) {
By("checking nodes have capacity present")
observedNodeList, err = manager.GetNodesWithOS(os)
Expect(err).ToNot(HaveOccurred())
for _, node := range observedNodeList.Items {
_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
if !found {
return false, nil
err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.ResourceCreationTimeout, true,
func(ctx context.Context) (bool, error) {
By("checking nodes have capacity present")
observedNodeList, err = manager.GetNodesWithOS(os)
Expect(err).ToNot(HaveOccurred())
for _, node := range observedNodeList.Items {
_, found := node.Status.Allocatable[v1.ResourceName(expectedResource)]
if !found {
return false, nil
}
}
}
return true, nil
})
return true, nil
})
Expect(err).ToNot(HaveOccurred())
return observedNodeList
}

// VerifyCNINode checks if the number of CNINodes is equal to number of nodes in the cluster, and verifies 1:1 mapping between CNINode and Node objects
// Returns nil if count and 1:1 mapping exists, else returns error
func VerifyCNINode(manager Manager) error {
var cniNodeList *cninode.CNINodeList
var nodeList *v1.NodeList
var err error
By("checking number of CNINodes match number of nodes in the cluster")
err = wait.PollUntilContextTimeout(context.Background(), utils.PollIntervalShort, utils.PollTimeout, true,
func(ctx context.Context) (bool, error) {
if cniNodeList, err = manager.GetCNINodeList(); err != nil {
return false, nil
}
if nodeList, err = manager.GetNodeList(); err != nil {
return false, nil
}
if len(nodeList.Items) != len(cniNodeList.Items) {
return false, nil
}
return true, nil
})
if err != nil {
return fmt.Errorf("number of CNINodes does not match number of nodes in the cluster")
}
By("checking CNINode list matches node list")
nameMatched := true
for _, node := range nodeList.Items {
if !lo.ContainsBy(cniNodeList.Items, func(cniNode cninode.CNINode) bool {
return cniNode.Name == node.Name
}) {
nameMatched = false
}
}
if !nameMatched {
return fmt.Errorf("CNINode list does not match node list")
}
return nil
}
1 change: 1 addition & 0 deletions test/framework/utils/poll.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const (
PollIntervalShort = 2 * time.Second
PollIntervalMedium = 10 * time.Second
PollIntervalLong = 20 * time.Second
PollTimeout = 30 * time.Second
// ResourceCreationTimeout is the number of seconds till the controller waits
// for the resource creation to complete
ResourceCreationTimeout = 120 * time.Second
Expand Down
50 changes: 50 additions & 0 deletions test/integration/cninode/cninode_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
// http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package cninode_test

import (
"testing"

"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework"
"github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/node"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestCNINode(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "CNINode Test Suite")
}

var frameWork *framework.Framework
var _ = BeforeSuite(func() {
By("creating a framework")
frameWork = framework.New(framework.GlobalOptions)

By("verify at least 2 nodes are available")
nodeList, err := frameWork.NodeManager.GetNodeList()
Expect(err).ToNot(HaveOccurred())
Expect(len(nodeList.Items)).To(BeNumerically(">", 1))

By("verify CNINode count")
err = node.VerifyCNINode(frameWork.NodeManager)
Expect(err).ToNot(HaveOccurred())
})

// Verify CNINode count before and after test remains same
var _ = AfterSuite(func() {
By("verify CNINode count")
err := node.VerifyCNINode(frameWork.NodeManager)
Expect(err).ToNot(HaveOccurred())
})
Loading

0 comments on commit 21961a7

Please sign in to comment.