diff --git a/pkg/aws/ec2/api/cleanup/eni_cleanup.go b/pkg/aws/ec2/api/cleanup/eni_cleanup.go index 0188e509..723fc92e 100644 --- a/pkg/aws/ec2/api/cleanup/eni_cleanup.go +++ b/pkg/aws/ec2/api/cleanup/eni_cleanup.go @@ -161,7 +161,7 @@ func (e *ENICleaner) DeleteLeakedResources() error { } else { // Seeing the ENI for the first time, add it to the new list of available network interfaces availableENIs[*nwInterface.NetworkInterfaceId] = struct{}{} - e.Log.V(1).Info("adding eni to to the map of available ENIs, will be removed if present in "+ + e.Log.Info("adding eni to to the map of available ENIs, will be removed if present in "+ "next run too", "id", *nwInterface.NetworkInterfaceId) } } diff --git a/test/framework/options.go b/test/framework/options.go index c4bdbe94..1b7b82c4 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -15,6 +15,8 @@ package framework import ( "flag" + "os" + "strings" "github.com/pkg/errors" "k8s.io/client-go/tools/clientcmd" @@ -32,6 +34,7 @@ type Options struct { AWSRegion string AWSVPCID string ReleasedImageVersion string + ClusterRoleArn string } func (options *Options) BindFlags() { @@ -40,6 +43,7 @@ func (options *Options) BindFlags() { flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`) flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`) flag.StringVar(&options.ReleasedImageVersion, "latest-released-rc-image-tag", "v1.1.3", `VPC RC latest released image`) + flag.StringVar(&options.ClusterRoleArn, "cluster-role-arn", "", "EKS Cluster role ARN") } func (options *Options) Validate() error { @@ -58,5 +62,10 @@ func (options *Options) Validate() error { if len(options.ReleasedImageVersion) == 0 { return errors.Errorf("%s must be set!", "latest-released-rc-image-tag") } + dir, err := os.Executable() + if err == nil && len(options.ClusterRoleArn) == 0 && strings.Contains(dir, "ec2api") { + return errors.Errorf("%s must be set when running ec2api tests", "cluster-role-arn") + } + return nil } diff --git a/test/framework/resource/aws/ec2/manager.go b/test/framework/resource/aws/ec2/manager.go index bbfa69d4..5c086a0c 100644 --- a/test/framework/resource/aws/ec2/manager.go +++ b/test/framework/resource/aws/ec2/manager.go @@ -19,6 +19,7 @@ import ( "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" @@ -240,3 +241,42 @@ func (d *Manager) GetPrivateIPv4AddressAndPrefix(instanceID string) ([]string, [ return secondaryIPAddresses, ipV4Prefixes, err } + +func (d *Manager) CreateAndAttachNetworkInterface(subnetID, instanceID, instanceType string) (string, error) { + createENIOp, err := d.ec2Client.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ + SubnetId: aws.String(subnetID), + Description: aws.String("VPC-Resource-Controller integration test ENI"), + }) + if err != nil { + return "", err + } + nwInterfaceID := *createENIOp.NetworkInterface.NetworkInterfaceId + // for test just use the max index - 2 (as trunk maybe attached to max index) + indexID := vpc.Limits[instanceType].NetworkCards[0].MaximumNetworkInterfaces - 2 + _, err = d.ec2Client.AttachNetworkInterface(&ec2.AttachNetworkInterfaceInput{ + InstanceId: aws.String(instanceID), + NetworkInterfaceId: aws.String(nwInterfaceID), + DeviceIndex: aws.Int64(indexID), + }) + return nwInterfaceID, err +} + +func (d *Manager) TerminateInstances(instanceID string) error { + _, err := d.ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ + InstanceIds: []*string{&instanceID}, + }) + return err +} + +func (d *Manager) DescribeNetworkInterface(nwInterfaceID string) error { + _, err := d.ec2Client.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: []*string{&nwInterfaceID}, + }) + return err +} +func (d *Manager) DeleteNetworkInterface(nwInterfaceID string) error { + _, err := d.ec2Client.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(nwInterfaceID), + }) + return err +} diff --git a/test/integration/ec2api/ec2api_suite_test.go b/test/integration/ec2api/ec2api_suite_test.go new file mode 100644 index 00000000..1183cc7d --- /dev/null +++ b/test/integration/ec2api/ec2api_suite_test.go @@ -0,0 +1,46 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +package ec2api_test + +import ( + "testing" + + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestEc2api(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "EC2API Suite") +} + +var frameWork *framework.Framework +var nodeListLen int +var _ = BeforeSuite(func() { + By("creating a framework") + frameWork = framework.New(framework.GlobalOptions) + By("verify node count before test") + nodeList, err := frameWork.NodeManager.GetNodesWithOS(config.OSLinux) + Expect(err).ToNot(HaveOccurred()) + nodeListLen = len(nodeList.Items) + Expect(nodeListLen).To(BeNumerically(">", 1)) +}) + +var _ = AfterSuite(func() { + nodeList, err := frameWork.NodeManager.GetNodesWithOS(config.OSLinux) + Expect(err).ToNot(HaveOccurred()) + By("verifying node count after test is unchanged") + Expect(len(nodeList.Items)).To(Equal(nodeListLen)) +}) diff --git a/test/integration/ec2api/ec2api_test.go b/test/integration/ec2api/ec2api_test.go new file mode 100644 index 00000000..bca12799 --- /dev/null +++ b/test/integration/ec2api/ec2api_test.go @@ -0,0 +1,95 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +package ec2api_test + +import ( + "strings" + "time" + + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" + "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// requires AmazonEKSVPCResourceController policy to be attached to the EKS cluster role +var _ = Describe("[LOCAL] Test IAM permissions for EC2 API calls", func() { + var instanceID string + var subnetID string + var instanceType string + var nwInterfaceID string + var err error + BeforeEach(func() { + By("getting instance details") + nodeList, err := frameWork.NodeManager.GetNodesWithOS(config.OSLinux) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeList.Items).ToNot(BeEmpty()) + instanceID = frameWork.NodeManager.GetInstanceID(&nodeList.Items[0]) + ec2Instance, err := frameWork.EC2Manager.GetInstanceDetails(instanceID) + Expect(err).ToNot(HaveOccurred()) + subnetID = *ec2Instance.SubnetId + instanceType = *ec2Instance.InstanceType + + }) + AfterEach(func() { + By("deleting test interface") + err = frameWork.EC2Manager.DeleteNetworkInterface(nwInterfaceID) + Expect(err).ToNot(HaveOccurred()) + }) + Describe("Test DeleteNetworkInterface permission", func() { + Context("when instance is terminated", func() { + It("it should only delete ENIs provisioned by the controller or vpc-cni", func() { + By("creating test ENI without eks:eni:owner tag and attach to EC2 instance") + nwInterfaceID, err = frameWork.EC2Manager.CreateAndAttachNetworkInterface(subnetID, instanceID, instanceType) + Expect(err).ToNot(HaveOccurred()) + By("terminating the instance and sleeping") + err = frameWork.EC2Manager.TerminateInstances(instanceID) + Expect(err).ToNot(HaveOccurred()) + // allow time for instance to be deleted and ENI to be available, new node to be ready + time.Sleep(utils.ResourceCreationTimeout) + By("verifying ENI is not deleted by controller") + err = frameWork.EC2Manager.DescribeNetworkInterface(nwInterfaceID) + Expect(err).ToNot(HaveOccurred()) + }) + }) + }) + Describe("Test CreateNetworkInterfacePermission permission", func() { + Context("when assuming EKS cluster role", func() { + It("it should not grant CreateNetworkInterfacePermission on ENIs without tag eks:eni:owner=eks-vpc-resource-controller", func() { + By("assuming EKS cluster role") + sess := session.Must(session.NewSession()) + creds := stscreds.NewCredentials(sess, frameWork.Options.ClusterRoleArn) + ec2Client := ec2.New(sess, &aws.Config{Credentials: creds}) + By("creating network interface") + nwInterfaceOp, err := ec2Client.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ + SubnetId: aws.String(subnetID), + Description: aws.String("VPC-Resource-Controller integration test ENI"), + }) + Expect(err).ToNot(HaveOccurred()) + nwInterfaceID = *nwInterfaceOp.NetworkInterface.NetworkInterfaceId + By("creating network interface permission") + _, err = ec2Client.CreateNetworkInterfacePermission(&ec2.CreateNetworkInterfacePermissionInput{ + AwsAccountId: aws.String(strings.Split(frameWork.Options.ClusterRoleArn, ":")[3]), + NetworkInterfaceId: aws.String(nwInterfaceID), + Permission: aws.String(ec2.InterfacePermissionTypeInstanceAttach), + }) + By("validating error occurred") + Expect(err).To(HaveOccurred()) + }) + }) + }) +})