diff --git a/.gitignore b/.gitignore index a9a2bd211d..cc0435f0dc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ portmap grpc-health-probe cni-metrics-helper coverage.txt -build/ \ No newline at end of file +build/ +vendor diff --git a/Makefile b/Makefile index 185b30507b..b52fcb5f7f 100644 --- a/Makefile +++ b/Makefile @@ -109,9 +109,9 @@ dist: all BUILD_MODE ?= -buildmode=pie build-linux: BUILD_FLAGS = $(BUILD_MODE) -ldflags '-s -w $(LDFLAGS)' build-linux: ## Build the VPC CNI plugin agent using the host's Go toolchain. - go build $(BUILD_FLAGS) -o aws-k8s-agent ./cmd/aws-k8s-agent - go build $(BUILD_FLAGS) -o aws-cni ./cmd/routed-eni-cni-plugin - go build $(BUILD_FLAGS) -o grpc-health-probe ./cmd/grpc-health-probe + go build -mod=mod $(BUILD_FLAGS) -o aws-k8s-agent ./cmd/aws-k8s-agent + go build -mod=mod $(BUILD_FLAGS) -o aws-cni ./cmd/routed-eni-cni-plugin + go build -mod=mod $(BUILD_FLAGS) -o grpc-health-probe ./cmd/grpc-health-probe # Build VPC CNI plugin & agent container image. docker: ## Build VPC CNI plugin & agent container image. @@ -136,7 +136,7 @@ docker-func-test: docker ## Run the built CNI container image to use in func # Run unit tests unit-test: export AWS_VPC_K8S_CNI_LOG_FILE=stdout unit-test: ## Run unit tests - go test -v -coverprofile=coverage.txt -covermode=atomic $(ALLPKGS) + go test -v -mod=mod -coverprofile=coverage.txt -covermode=atomic $(ALLPKGS) # Run unit tests with race detection (can only be run natively) unit-test-race: export AWS_VPC_K8S_CNI_LOG_FILE=stdout @@ -199,7 +199,7 @@ generate: # Generate eni-max-pods.txt file for EKS AMI generate-limits: GOOS= generate-limits: ## Generate limit file go code - go run scripts/gen_vpc_ip_limits.go + go run -mod=mod scripts/gen_vpc_ip_limits.go # Fetch the CNI plugins plugins: FETCH_VERSION=0.9.0 diff --git a/README.md b/README.md index c1da32eeac..be55d5457c 100644 --- a/README.md +++ b/README.md @@ -450,6 +450,30 @@ You can use the below command to enable `DISABLE_TCP_EARLY_DEMUX` to `true` - ``` kubectl patch daemonset aws-node -n kube-system -p '{"spec": {"template": {"spec": {"initContainers": [{"env":[{"name":"DISABLE_TCP_EARLY_DEMUX","value":"true"}],"name":"aws-vpc-cni-init"}]}}}}' ``` +--- + +`ENABLE_PREFIX_DELEGATION` (Since v1.9) + +Type: Boolean as a String + +Default: `false` + +To enable IPv4 prefix delegation on nitro instances. Setting `ENABLE_PREFIX_DELEGATION` flag toggle to `true` will start allocating a /28 prefix +instead of a secondary IP in the ENIs subnet. The total number of prefixes and private IP addresses will be less than the +limit on private IPs allowed by your instance. The current preview will support a single /28 prefix per ENI. Knob toggle while pods are running or if +ENIs are attached is not supported. On toggling the knob, node should be recycled to set the new kubelet max pods value. + +--- + +`WARM_PREFIX_TARGET` + +Type: Integer + +Default: None + +Specifies the number of free IPv4(/28) prefixes that the `ipamd` daemon should attempt to keep available for pod assignment on the node. +This environment variable overrides `WARM_ENI_TARGET`, `WARM_IP_TARGET` and `MINIMUM_IP_TARGET` and works when `ENABLE_PREFIX_DELEGATION` +is set to `true`. The current preview release will support a single /28 prefix per ENI hence setting this will cause additional ENIs to be allocated. ### ENI tags related to Allocation diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml index 602affeadb..7a471dcb8e 100644 --- a/config/master/aws-k8s-cni.yaml +++ b/config/master/aws-k8s-cni.yaml @@ -147,6 +147,8 @@ "value": "false" - "name": "ENABLE_POD_ENI" "value": "false" + - "name": "ENABLE_PREFIX_DELEGATION" + "value": "false" - "name": "MY_NODE_NAME" "valueFrom": "fieldRef": diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet index d25a27c040..279b8ba7ca 100644 --- a/config/master/manifests.jsonnet +++ b/config/master/manifests.jsonnet @@ -174,6 +174,7 @@ local awsnode = { DISABLE_INTROSPECTION: "false", DISABLE_METRICS: "false", ENABLE_POD_ENI: "false", + ENABLE_PREFIX_DELEGATION: "false", MY_NODE_NAME: { valueFrom: { fieldRef: {fieldPath: "spec.nodeName"}, diff --git a/go.mod b/go.mod index 451073a846..6e858e5dd7 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/google/btree v1.0.0 // indirect github.com/google/go-jsonnet v0.16.0 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect - github.com/google/gopacket v1.1.18 github.com/googleapis/gnostic v0.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect @@ -50,3 +49,8 @@ require ( k8s.io/cri-api v0.0.0-20191107035106-03d130a7dc28 k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect ) + +//aws-sdk-go with prefix delegation is not yet GA. Hence replacing the SDK with the +//vendored copy generated by replacing the api-2.json, docs-2.json, examples-1.json and paginators-1.json files +//with the new model in models/service/ec2 +replace github.com/aws/aws-sdk-go => ./vendor/github.com/aws/aws-sdk-go diff --git a/go.sum b/go.sum index ce85b85170..34e2bad780 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,6 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/aws/aws-sdk-go v1.35.27 h1:F0dUW+kouzchjt4X6kYfYMw1YtQPkA4pihpCDqQMrq8= -github.com/aws/aws-sdk-go v1.35.27/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/aws/aws-sdk-go v1.37.23 h1:bO80NcSmRv52w+GFpBegoLdlP/Z0OwUqQ9bbeCLCy/0= -github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -19,8 +15,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.8.7 h1:bU7QieuAp+sACI2vCzESJ3FoT860urYP+lThyZkb/2M= -github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containernetworking/plugins v0.9.0 h1:c+1gegKhR7+d0Caum9pEHugZlyhXPOG6v3V6xJgIGCI= github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg= github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= @@ -61,11 +55,8 @@ github.com/golang/mock v1.4.1 h1:ocYkMQY5RrXTYgXl7ICpV0IXwlEQGwKIsery4gyXa1U= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -79,14 +70,13 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-jsonnet v0.16.0 h1:Nb4EEOp+rdeGGyB1rQ5eisgSAqrTnhf9ip+X6lzZbY0= github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= -github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= @@ -131,18 +121,15 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/operator-framework/operator-sdk v0.0.7 h1:feujqHLhibLBbDVrSAFswpSzTVS5mEuarvywJ079mYE= github.com/operator-framework/operator-sdk v0.0.7/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA= @@ -181,14 +168,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -200,7 +181,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= @@ -225,14 +205,11 @@ golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -251,10 +228,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -267,6 +241,8 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637 h1:O5hKNaGxIT4A8OTMnuh6UpmBdI3SAPxlZ3g0olDrJVM= golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -336,6 +312,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/misc/eni-max-pods.txt b/misc/eni-max-pods.txt index 4ac9aa86b3..258f9babfd 100644 --- a/misc/eni-max-pods.txt +++ b/misc/eni-max-pods.txt @@ -11,7 +11,7 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. # -# This file was generated at 2021-01-24T12:55:41+11:00 +# This file was generated at 2021-04-26T18:05:13Z # # Mapping is calculated from AWS EC2 API using the following formula: # * First IP on each ENI is not used for pods @@ -229,6 +229,7 @@ m5dn.2xlarge 58 m5dn.4xlarge 234 m5dn.8xlarge 234 m5dn.large 29 +m5dn.metal 737 m5dn.xlarge 58 m5n.12xlarge 234 m5n.16xlarge 737 @@ -237,6 +238,7 @@ m5n.2xlarge 58 m5n.4xlarge 234 m5n.8xlarge 234 m5n.large 29 +m5n.metal 737 m5n.xlarge 58 m5zn.12xlarge 737 m5zn.2xlarge 58 @@ -333,6 +335,7 @@ r5dn.2xlarge 58 r5dn.4xlarge 234 r5dn.8xlarge 234 r5dn.large 29 +r5dn.metal 737 r5dn.xlarge 58 r5n.12xlarge 234 r5n.16xlarge 737 @@ -341,6 +344,7 @@ r5n.2xlarge 58 r5n.4xlarge 234 r5n.8xlarge 234 r5n.large 29 +r5n.metal 737 r5n.xlarge 58 r6g.12xlarge 234 r6g.16xlarge 737 @@ -402,6 +406,15 @@ x1e.32xlarge 234 x1e.4xlarge 58 x1e.8xlarge 58 x1e.xlarge 29 +x2gd.12xlarge 234 +x2gd.16xlarge 737 +x2gd.2xlarge 58 +x2gd.4xlarge 234 +x2gd.8xlarge 234 +x2gd.large 29 +x2gd.medium 8 +x2gd.metal 737 +x2gd.xlarge 58 z1d.12xlarge 737 z1d.2xlarge 58 z1d.3xlarge 234 diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index b92875247b..f95dc4023f 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -117,6 +117,9 @@ type APIs interface { // GetIPv4sFromEC2 returns the IPv4 addresses for a given ENI GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error) + // GetIPv4PrefixesFromEC2 returns the IPv4 addresses for a given ENI + GetIPv4PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv4PrefixSpecification, err error) + // DescribeAllENIs calls EC2 and returns a fully populated DescribeAllENIsResult struct and an error DescribeAllENIs() (DescribeAllENIsResult, error) @@ -127,7 +130,10 @@ type APIs interface { AllocIPAddresses(eniID string, numIPs int) error // DeallocIPAddresses deallocates the list of IP addresses from a ENI - DeallocIPAddresses(eniID string, ips []string) error + DeallocIPAddresses(eniID string, ips []string, igonoreCachedPDflag bool) error + + // DeallocPrefixAddresses deallocates the list of IP addresses from a ENI + DeallocPrefixAddresses(eniID string, ips []string) error // GetVPCIPv4CIDRs returns VPC's CIDRs from instance metadata GetVPCIPv4CIDRs() ([]string, error) @@ -164,26 +170,32 @@ type APIs interface { //RefreshSGIDs RefreshSGIDs(mac string) error + + //GetInstanceHypervisorFamily returns the hypervisor family for the instance + GetInstanceHypervisorFamily() (string, error) + + //GetInstanceType returns the EC2 instance type + GetInstanceType() string } // EC2InstanceMetadataCache caches instance metadata type EC2InstanceMetadataCache struct { // metadata info - securityGroups StringSet - subnetID string - localIPv4 net.IP - instanceID string - instanceType string - primaryENI string - primaryENImac string - availabilityZone string - region string - unmanagedENIs StringSet - useCustomNetworking bool - cniunmanagedENIs StringSet - - imds TypedIMDS - ec2SVC ec2wrapper.EC2 + securityGroups StringSet + subnetID string + localIPv4 net.IP + instanceID string + instanceType string + primaryENI string + primaryENImac string + availabilityZone string + region string + unmanagedENIs StringSet + useCustomNetworking bool + cniunmanagedENIs StringSet + useIPv4PrefixDelegation bool + imds TypedIMDS + ec2SVC ec2wrapper.EC2 } // ENIMetadata contains information about an ENI @@ -202,12 +214,16 @@ type ENIMetadata struct { // The ip addresses allocated for the network interface IPv4Addresses []*ec2.NetworkInterfacePrivateIpAddress + + // IPv4 Prefixes allocated for the network interface + IPv4Prefixes []*ec2.Ipv4PrefixSpecification } // InstanceTypeLimits keeps track of limits for an instance type type InstanceTypeLimits struct { - ENILimit int - IPv4Limit int + ENILimit int + IPv4Limit int + HypervisorType string } // PrimaryIPv4Address returns the primary IP of this node @@ -315,8 +331,28 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) return result, nil } +func (cache *EC2InstanceMetadataCache) getNetworkInterfacesWithContext(ctx context.Context, eniID string) (*ec2.DescribeNetworkInterfacesOutput, error) { + eniIds := []*string{aws.String(eniID)} + input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} + + start := time.Now() + result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input) + awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { + return nil, ErrENINotFound + } + } + awsAPIErrInc("DescribeNetworkInterfaces", err) + log.Errorf("Failed to get ENI %s information from EC2 control plane %v", eniIds, err) + return nil, errors.Wrap(err, "failed to describe network interface") + } + return result, nil +} + // New creates an EC2InstanceMetadataCache -func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { +func New(useCustomNetworking, useIPv4PrefixDelegation bool) (*EC2InstanceMetadataCache, error) { //ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run ctx := context.Background() @@ -338,7 +374,10 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { log.Debugf("Discovered region: %s", cache.region) cache.useCustomNetworking = useCustomNetworking - log.Infof("Custom networking %v", cache.useCustomNetworking) + log.Infof("Custom networking enabled %v", cache.useCustomNetworking) + + cache.useIPv4PrefixDelegation = useIPv4PrefixDelegation + log.Infof("Prefix Delegation enabled %v", cache.useIPv4PrefixDelegation) awsCfg := aws.NewConfig().WithRegion(region) sess = sess.Copy(awsCfg) @@ -550,12 +589,37 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat } } + var ec2ipv4Prefixes []*ec2.Ipv4PrefixSpecification + //Get prefix on primary ENI when custom networking is enabled is not needed. Everytime we + //call attached ENIs, the call will return prefix not found in the logs and that will pollute + //ipamd.log hence skipping. + + //Prefix get is taking more time, so adding a check for preview. Will remove this for GA. + if cache.useIPv4PrefixDelegation { + start := time.Now() + log.Debugf("Querying for prefix") + if (eniMAC == primaryMAC && !cache.useCustomNetworking) || (eniMAC != primaryMAC) { + imdsIPv4Prefixes, err := cache.imds.GetLocalIPv4Prefixes(ctx, eniMAC) + if err != nil { + return ENIMetadata{}, err + } + for _, ipv4prefix := range imdsIPv4Prefixes { + ec2ipv4Prefixes = append(ec2ipv4Prefixes, &ec2.Ipv4PrefixSpecification{ + Ipv4Prefix: aws.String(ipv4prefix.String()), + }) + } + } + elapsed := time.Since(start) + log.Debugf("Time taken to return prefix query %s", elapsed) + } + return ENIMetadata{ ENIID: eniID, MAC: eniMAC, DeviceNumber: deviceNum, SubnetIPv4CIDR: cidr.String(), IPv4Addresses: ec2ip4s, + IPv4Prefixes: ec2ipv4Prefixes, }, nil } @@ -886,22 +950,9 @@ func (cache *EC2InstanceMetadataCache) freeENI(eniName string, sleepDelayAfterDe // getENIAttachmentID calls EC2 to fetch the attachmentID of a given ENI func (cache *EC2InstanceMetadataCache) getENIAttachmentID(eniID string) (*string, error) { - eniIds := make([]*string, 0) - eniIds = append(eniIds, aws.String(eniID)) - input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} - - start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input) - awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + result, err := cache.getNetworkInterfacesWithContext(context.Background(), eniID) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { - return nil, ErrENINotFound - } - } - awsAPIErrInc("DescribeNetworkInterfaces", err) - log.Errorf("Failed to get ENI %s information from EC2 control plane %v", eniID, err) - return nil, errors.Wrap(err, "failed to describe network interface") + return nil, err } // Shouldn't happen, but let's be safe if len(result.NetworkInterfaces) == 0 { @@ -948,22 +999,9 @@ func (cache *EC2InstanceMetadataCache) deleteENI(eniName string, maxBackoffDelay // GetIPv4sFromEC2 calls EC2 and returns a list of all addresses on the ENI func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error) { - eniIds := make([]*string, 0) - eniIds = append(eniIds, aws.String(eniID)) - input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} - - start := time.Now() - result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input) - awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + result, err := cache.getNetworkInterfacesWithContext(context.Background(), eniID) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { - return nil, ErrENINotFound - } - } - awsAPIErrInc("DescribeNetworkInterfaces", err) - log.Errorf("Failed to get ENI %s information from EC2 control plane %v", eniID, err) - return nil, errors.Wrap(err, "failed to describe network interface") + return nil, err } // Shouldn't happen, but let's be safe @@ -975,6 +1013,22 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [ return firstNI.PrivateIpAddresses, nil } +// GetIPv4PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI +func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv4PrefixSpecification, err error) { + result, err := cache.getNetworkInterfacesWithContext(context.Background(), eniID) + if err != nil { + return nil, err + } + + // Shouldn't happen, but let's be safe + if len(result.NetworkInterfaces) == 0 { + return nil, ErrNoNetworkInterfaces + } + returnedENI := result.NetworkInterfaces[0] + + return returnedENI.Ipv4Prefixes, nil +} + // DescribeAllENIs calls EC2 to refresh the ENIMetadata and tags for all attached ENIs func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, error) { // Fetch all local ENI info from metadata @@ -1198,6 +1252,22 @@ func (cache *EC2InstanceMetadataCache) GetENILimit() (int, error) { return eniLimits.ENILimit, nil } +// GetInstanceHypervisorFamily return hypervior of EC2 instance type +func (cache *EC2InstanceMetadataCache) GetInstanceHypervisorFamily() (string, error) { + eniLimits, ok := InstanceNetworkingLimits[cache.instanceType] + if !ok { + log.Errorf("Failed to get hypervisor info due to unknown instance type %s", cache.instanceType) + return "", errors.New(UnknownInstanceType) + } + log.Debugf("Instance hypervisor family %s", eniLimits.HypervisorType) + return eniLimits.HypervisorType, nil +} + +// GetInstanceType return EC2 instance type +func (cache *EC2InstanceMetadataCache) GetInstanceType() string { + return cache.instanceType +} + // AllocIPAddresses allocates numIPs of IP address on an ENI func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int) error { var needIPs = numIPs @@ -1218,9 +1288,21 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int } log.Infof("Trying to allocate %d IP addresses on ENI %s", needIPs, eniID) - input := &ec2.AssignPrivateIpAddressesInput{ - NetworkInterfaceId: aws.String(eniID), - SecondaryPrivateIpAddressCount: aws.Int64(int64(needIPs)), + + input := &ec2.AssignPrivateIpAddressesInput{} + + if cache.useIPv4PrefixDelegation { + needPrefixes := needIPs + input = &ec2.AssignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4PrefixCount: aws.Int64(int64(needPrefixes)), + } + + } else { + input = &ec2.AssignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + SecondaryPrivateIpAddressCount: aws.Int64(int64(needIPs)), + } } start := time.Now() @@ -1237,17 +1319,21 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int return errors.Wrap(err, "allocate IP address: failed to allocate a private IP address") } if output != nil { - log.Infof("Allocated %d private IP addresses", len(output.AssignedPrivateIpAddresses)) + if cache.useIPv4PrefixDelegation { + log.Infof("Allocated %d private IP prefixes", len(output.AssignedIpv4Prefixes)) + } else { + log.Infof("Allocated %d private IP addresses", len(output.AssignedPrivateIpAddresses)) + } } return nil } // WaitForENIAndIPsAttached waits until the ENI has been attached and the secondary IPs have been added -func (cache *EC2InstanceMetadataCache) WaitForENIAndIPsAttached(eni string, wantedSecondaryIPs int) (eniMetadata ENIMetadata, err error) { - return cache.waitForENIAndIPsAttached(eni, wantedSecondaryIPs, maxENIBackoffDelay) +func (cache *EC2InstanceMetadataCache) WaitForENIAndIPsAttached(eni string, wantedSecondaryIPsOrPrefixes int) (eniMetadata ENIMetadata, err error) { + return cache.waitForENIAndIPsAttached(eni, wantedSecondaryIPsOrPrefixes, maxENIBackoffDelay) } -func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, wantedSecondaryIPs int, maxBackoffDelay time.Duration) (eniMetadata ENIMetadata, err error) { +func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, wantedSecondaryIPsOrPrefixes int, maxBackoffDelay time.Duration) (eniMetadata ENIMetadata, err error) { start := time.Now() attempt := 0 // Wait until the ENI shows up in the instance metadata service and has at least some secondary IPs @@ -1262,15 +1348,23 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want for _, returnedENI := range enis { if eni == returnedENI.ENIID { // Check how many Secondary IPs have been attached - eniIPCount := len(returnedENI.IPv4Addresses) - if eniIPCount <= 1 { + var eniIPCount int + if cache.useIPv4PrefixDelegation { + eniIPCount = len(returnedENI.IPv4Prefixes) + } else { + //Ignore primary IP of the ENI + eniIPCount = len(returnedENI.IPv4Addresses) - 1 + } + + if eniIPCount < 1 { log.Debugf("No secondary IPv4 addresses available yet on ENI %s", returnedENI.ENIID) return ErrNoSecondaryIPsFound } + // At least some are attached eniMetadata = returnedENI // ipsToAllocate will be at most 1 less then the IP limit for the ENI because of the primary IP - if eniIPCount > wantedSecondaryIPs { + if (eniIPCount > wantedSecondaryIPsOrPrefixes && !cache.useIPv4PrefixDelegation) || (eniIPCount >= wantedSecondaryIPsOrPrefixes && cache.useIPv4PrefixDelegation) { return nil } return ErrAllSecondaryIPsNotFound @@ -1283,9 +1377,13 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want if err != nil { // If we have at least 1 Secondary IP, by now return what we have without an error if err == ErrAllSecondaryIPsNotFound { - if len(eniMetadata.IPv4Addresses) > 1 { + if !cache.useIPv4PrefixDelegation && len(eniMetadata.IPv4Addresses) > 1 { // We have some Secondary IPs, return the ones we have - log.Warnf("This ENI only has %d IP addresses, we wanted %d", len(eniMetadata.IPv4Addresses), wantedSecondaryIPs) + log.Warnf("This ENI only has %d IP addresses, we wanted %d", len(eniMetadata.IPv4Addresses), wantedSecondaryIPsOrPrefixes) + return eniMetadata, nil + } else if cache.useIPv4PrefixDelegation && len(eniMetadata.IPv4Prefixes) > 1 { + // We have some prefixes, return the ones we have + log.Warnf("This ENI only has %d Prefixes, we wanted %d", len(eniMetadata.IPv4Prefixes), wantedSecondaryIPsOrPrefixes) return eniMetadata, nil } } @@ -1296,16 +1394,42 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want } // DeallocIPAddresses allocates numIPs of IP address on an ENI -func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []string) error { +func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []string, ignoreCachedPDflag bool) error { log.Infof("Trying to unassign the following IPs %s from ENI %s", ips, eniID) - var ipsInput []*string - for _, ip := range ips { - ipsInput = append(ipsInput, aws.String(ip)) + ipsInput := aws.StringSlice(ips) + + input := &ec2.UnassignPrivateIpAddressesInput{} + if !cache.useIPv4PrefixDelegation || ignoreCachedPDflag { + input = &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + PrivateIpAddresses: ipsInput, + } + } else { + input = &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4Prefixes: ipsInput, + } } + start := time.Now() + _, err := cache.ec2SVC.UnassignPrivateIpAddressesWithContext(context.Background(), input) + awsAPILatency.WithLabelValues("UnassignPrivateIpAddresses", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + awsAPIErrInc("UnassignPrivateIpAddresses", err) + log.Errorf("Failed to deallocate a private IP address %v", err) + return errors.Wrap(err, fmt.Sprintf("deallocate IP addresses: failed to deallocate private IP addresses: %s", ips)) + } + return nil +} + +// DeallocPrefixAddresses allocates numIPs of IP address on an ENI +func (cache *EC2InstanceMetadataCache) DeallocPrefixAddresses(eniID string, ips []string) error { + log.Infof("Trying to unassign the following IPs %s from ENI %s", ips, eniID) + ipsInput := aws.StringSlice(ips) + input := &ec2.UnassignPrivateIpAddressesInput{ NetworkInterfaceId: aws.String(eniID), - PrivateIpAddresses: ipsInput, + Ipv4Prefixes: ipsInput, } start := time.Now() diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index 69b253d731..c00aefe898 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -48,6 +48,7 @@ const ( metadataInterface = "/interface-id" metadataSubnetCIDR = "/subnet-ipv4-cidr-block" metadataIPv4s = "/local-ipv4s" + metadataIPv4Prefixes = "/ipv4-prefix" az = "us-east-1a" localIP = "10.0.0.10" @@ -65,8 +66,10 @@ const ( eniAttachID = "eni-attach-beb21856" eni1Device = "0" eni1PrivateIP = "10.0.0.1" + eni1Prefix = "10.0.1.0/28" eni2Device = "1" eni2PrivateIP = "10.0.0.2" + eni2Prefix = "10.0.2.0/28" eni2ID = "eni-12341234" metadataVPCIPv4CIDRs = "192.168.0.0/16 100.66.0.0/1" ) @@ -95,6 +98,31 @@ func testMetadata(overrides map[string]interface{}) FakeIMDS { return FakeIMDS(data) } +func testMetadataWithPrefixes(overrides map[string]interface{}) FakeIMDS { + data := map[string]interface{}{ + metadataAZ: az, + metadataLocalIP: localIP, + metadataInstanceID: instanceID, + metadataInstanceType: instanceType, + metadataMAC: primaryMAC, + metadataMACPath: primaryMAC, + metadataMACPath + primaryMAC + metadataDeviceNum: eni1Device, + metadataMACPath + primaryMAC + metadataInterface: primaryeniID, + metadataMACPath + primaryMAC + metadataSGs: sgs, + metadataMACPath + primaryMAC + metadataIPv4s: eni1PrivateIP, + metadataMACPath + primaryMAC + metadataIPv4Prefixes: eni1Prefix, + metadataMACPath + primaryMAC + metadataSubnetID: subnetID, + metadataMACPath + primaryMAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + primaryMAC + metadataVPCcidrs: metadataVPCIPv4CIDRs, + } + + for k, v := range overrides { + data[k] = v + } + + return FakeIMDS(data) +} + func setup(t *testing.T) (*gomock.Controller, *mock_ec2wrapper.MockEC2) { ctrl := gomock.NewController(t) @@ -164,6 +192,23 @@ func TestGetAttachedENIs(t *testing.T) { } } +func TestGetAttachedENIsWithPrefixes(t *testing.T) { + mockMetadata := testMetadata(map[string]interface{}{ + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + eni2MAC + metadataIPv4s: eni2PrivateIP, + metadataMACPath + eni2MAC + metadataIPv4Prefixes: eni2Prefix, + }) + + ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} + ens, err := ins.GetAttachedENIs() + if assert.NoError(t, err) { + assert.Equal(t, len(ens), 2) + } +} + func TestAWSGetFreeDeviceNumberOnErr(t *testing.T) { ctrl, mockEC2 := setup(t) defer ctrl.Finish() @@ -683,6 +728,43 @@ func TestAllocIPAddressesAlreadyFull(t *testing.T) { assert.NoError(t, err) } +func TestAllocPrefixAddresses(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + + //Allocate 1 prefix for the ENI + input := &ec2.AssignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4PrefixCount: aws.Int64(1), + } + mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), input, gomock.Any()).Return(nil, nil) + + ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", ipv4PrefixDelegation: true} + err := ins.AllocIPAddresses(eniID, 1) + assert.NoError(t, err) + + // Adding 0 should do nothing + err = ins.AllocIPAddresses(eniID, 0) + assert.NoError(t, err) +} + +func TestAllocPrefixesAlreadyFull(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + // The required Prefixes (1) is the ENI's limit(1) + input := &ec2.AssignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv4PrefixCount: aws.Int64(1), + } + ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "t3.xlarge", ipv4PrefixDelegation: true} + + retErr := awserr.New("PrivateIpAddressLimitExceeded", "Too many IPs already allocated", nil) + mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), input, gomock.Any()).Return(nil, retErr) + // If EC2 says that all IPs are already attached, we do nothing + err := ins.AllocIPAddresses(eniID, 1) + assert.NoError(t, err) +} + func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_OneResult(t *testing.T) { ctrl, mockEC2 := setup(t) defer ctrl.Finish() @@ -763,15 +845,17 @@ func Test_badENIID(t *testing.T) { func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { type args struct { - eni string - foundSecondaryIPs int - wantedSecondaryIPs int - maxBackoffDelay time.Duration - times int + eni string + foundSecondaryIPs int + wantedSecondaryIPs int + maxBackoffDelay time.Duration + times int + enableIpv4PrefixDelegation bool } eni1Metadata := ENIMetadata{ ENIID: eniID, IPv4Addresses: nil, + IPv4Prefixes: nil, } isPrimary := true notPrimary := false @@ -795,6 +879,7 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { PrivateIpAddress: &secondaryIP2, }, }, + IPv4Prefixes: make([]*ec2.Ipv4PrefixSpecification, 0), } eniList := []ENIMetadata{eni1Metadata, eni2Metadata} tests := []struct { @@ -836,6 +921,80 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { } } +func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { + type args struct { + eni string + foundPrefixes int + wantedSecondaryIPs int + maxBackoffDelay time.Duration + times int + } + eni1Metadata := ENIMetadata{ + ENIID: eniID, + IPv4Addresses: nil, + IPv4Prefixes: nil, + } + isPrimary := true + primaryIP := eni2PrivateIP + prefixIP := eni2Prefix + eni2Metadata := ENIMetadata{ + ENIID: eni2ID, + MAC: eni2MAC, + DeviceNumber: 1, + SubnetIPv4CIDR: subnetCIDR, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + Primary: &isPrimary, + PrivateIpAddress: &primaryIP, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &prefixIP, + }, + }, + } + eniList := []ENIMetadata{eni1Metadata, eni2Metadata} + tests := []struct { + name string + args args + wantEniMetadata ENIMetadata + wantErr bool + }{ + {"Test wait success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: 1}, eniList[1], false}, + {"Test partial success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries}, eniList[1], false}, + {"Test wait fail", args{eni: eni2ID, foundPrefixes: 0, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries}, ENIMetadata{}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl, mockEC2 := setup(t) + defer ctrl.Finish() + eniIPs := eni2PrivateIP + eniPrefixes := eni2Prefix + if tt.args.foundPrefixes == 0 { + eniPrefixes = "" + } + fmt.Println("eniips", eniIPs) + mockMetadata := testMetadata(map[string]interface{}{ + metadataMACPath: primaryMAC + " " + eni2MAC, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + eni2MAC + metadataIPv4s: eniIPs, + metadataMACPath + eni2MAC + metadataIPv4Prefixes: eniPrefixes, + }) + cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, ipv4PrefixDelegation: true} + gotEniMetadata, err := cache.waitForENIAndIPsAttached(tt.args.eni, tt.args.wantedSecondaryIPs, tt.args.maxBackoffDelay) + if (err != nil) != tt.wantErr { + t.Errorf("waitForENIAndIPsAttached() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotEniMetadata, tt.wantEniMetadata) { + t.Errorf("waitForENIAndIPsAttached() gotEniMetadata = %v, want %v", gotEniMetadata, tt.wantEniMetadata) + } + }) + } +} func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { mockMetadata := testMetadata(nil) ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go index 39462e9bb6..2016dac99d 100644 --- a/pkg/awsutils/imds.go +++ b/pkg/awsutils/imds.go @@ -183,6 +183,16 @@ func (imds TypedIMDS) GetLocalIPv4s(ctx context.Context, mac string) ([]net.IP, return imds.getIPs(ctx, key) } +// GetLocalIPv4Prefixes returns the IPv4 prefixes delegated to this interface +func (imds TypedIMDS) GetLocalIPv4Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/ipv4-prefix", mac) + prefixes, err := imds.getCIDRs(ctx, key) + if IsNotFound(err) { + return nil, nil + } + return prefixes, err +} + // GetIPv6s returns the IPv6 addresses associated with the interface. func (imds TypedIMDS) GetIPv6s(ctx context.Context, mac string) ([]net.IP, error) { key := fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac) diff --git a/pkg/awsutils/imds_test.go b/pkg/awsutils/imds_test.go index edd9c64794..31ea549cca 100644 --- a/pkg/awsutils/imds_test.go +++ b/pkg/awsutils/imds_test.go @@ -232,3 +232,19 @@ func TestGetVPCIPv6CIDRBlocks(t *testing.T) { assert.True(t, IsNotFound(err)) } } + +func TestGetLocalIPv4Prefixes(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/ipv4-prefix": `10.1.1.0/28`, + })} + + ips, err := f.GetLocalIPv4Prefixes(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, []net.IPNet{{IP: net.IPv4(10, 1, 1, 0), Mask: net.CIDRMask(28, 32)}}) + } + + ips, err = f.GetLocalIPv4Prefixes(context.TODO(), "00:00:de:ad:be:ef") + if assert.NoError(t, err) { + assert.ElementsMatch(t, ips, []net.IPNet{}) + } +} diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index 473a2881dc..c2e3c87f37 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -94,17 +94,31 @@ func (mr *MockAPIsMockRecorder) AllocIPAddresses(arg0, arg1 interface{}) *gomock } // DeallocIPAddresses mocks base method -func (m *MockAPIs) DeallocIPAddresses(arg0 string, arg1 []string) error { +func (m *MockAPIs) DeallocIPAddresses(arg0 string, arg1 []string, arg2 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeallocIPAddresses", arg0, arg1) + ret := m.ctrl.Call(m, "DeallocIPAddresses", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 } // DeallocIPAddresses indicates an expected call of DeallocIPAddresses -func (mr *MockAPIsMockRecorder) DeallocIPAddresses(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockAPIsMockRecorder) DeallocIPAddresses(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocIPAddresses), arg0, arg1, arg2) +} + +// DeallocPrefixAddresses mocks base method +func (m *MockAPIs) DeallocPrefixAddresses(arg0 string, arg1 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeallocPrefixAddresses", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeallocPrefixAddresses indicates an expected call of DeallocPrefixAddresses +func (mr *MockAPIsMockRecorder) DeallocPrefixAddresses(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeallocPrefixAddresses", reflect.TypeOf((*MockAPIs)(nil).DeallocPrefixAddresses), arg0, arg1) } // DescribeAllENIs mocks base method @@ -181,6 +195,21 @@ func (mr *MockAPIsMockRecorder) GetENILimit() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetENILimit", reflect.TypeOf((*MockAPIs)(nil).GetENILimit)) } +// GetIPv4PrefixesFromEC2 mocks base method +func (m *MockAPIs) GetIPv4PrefixesFromEC2(arg0 string) ([]*ec2.Ipv4PrefixSpecification, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIPv4PrefixesFromEC2", arg0) + ret0, _ := ret[0].([]*ec2.Ipv4PrefixSpecification) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIPv4PrefixesFromEC2 indicates an expected call of GetIPv4PrefixesFromEC2 +func (mr *MockAPIsMockRecorder) GetIPv4PrefixesFromEC2(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4PrefixesFromEC2), arg0) +} + // GetIPv4sFromEC2 mocks base method func (m *MockAPIs) GetIPv4sFromEC2(arg0 string) ([]*ec2.NetworkInterfacePrivateIpAddress, error) { m.ctrl.T.Helper() @@ -196,6 +225,35 @@ func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), arg0) } +// GetInstanceHypervisorFamily mocks base method +func (m *MockAPIs) GetInstanceHypervisorFamily() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInstanceHypervisorFamily") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetInstanceHypervisorFamily indicates an expected call of GetInstanceHypervisorFamily +func (mr *MockAPIsMockRecorder) GetInstanceHypervisorFamily() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstanceHypervisorFamily", reflect.TypeOf((*MockAPIs)(nil).GetInstanceHypervisorFamily)) +} + +// GetInstanceType mocks base method +func (m *MockAPIs) GetInstanceType() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInstanceType") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetInstanceType indicates an expected call of GetInstanceType +func (mr *MockAPIsMockRecorder) GetInstanceType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInstanceType", reflect.TypeOf((*MockAPIs)(nil).GetInstanceType)) +} + // GetLocalIPv4 mocks base method func (m *MockAPIs) GetLocalIPv4() net.IP { m.ctrl.T.Helper() diff --git a/pkg/awsutils/vpc_ip_resource_limit.go b/pkg/awsutils/vpc_ip_resource_limit.go index e953b77fd0..7d38147bf0 100644 --- a/pkg/awsutils/vpc_ip_resource_limit.go +++ b/pkg/awsutils/vpc_ip_resource_limit.go @@ -12,399 +12,412 @@ // permissions and limitations under the License. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2021-01-24T12:55:41+11:00 +// This file was generated at 2021-04-26T18:05:13Z package awsutils // InstanceNetworkingLimits contains a mapping from instance type to networking limits for the type. Documentation found at // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI var InstanceNetworkingLimits = map[string]InstanceTypeLimits{ - "a1.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "a1.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "a1.large": {ENILimit: 3, IPv4Limit: 10}, - "a1.medium": {ENILimit: 2, IPv4Limit: 4}, - "a1.metal": {ENILimit: 8, IPv4Limit: 30}, - "a1.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c1.medium": {ENILimit: 2, IPv4Limit: 6}, - "c1.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c3.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c3.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c3.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c3.large": {ENILimit: 3, IPv4Limit: 10}, - "c3.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c4.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c4.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c4.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c4.large": {ENILimit: 3, IPv4Limit: 10}, - "c4.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5.18xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5.9xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5.large": {ENILimit: 3, IPv4Limit: 10}, - "c5.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5a.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5a.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5a.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5a.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5a.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5a.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5a.large": {ENILimit: 3, IPv4Limit: 10}, - "c5a.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5a.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5ad.large": {ENILimit: 3, IPv4Limit: 10}, - "c5ad.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5ad.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5d.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5d.18xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5d.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5d.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5d.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5d.9xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5d.large": {ENILimit: 3, IPv4Limit: 10}, - "c5d.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5d.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5n.18xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c5n.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c5n.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5n.9xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c5n.large": {ENILimit: 3, IPv4Limit: 10}, - "c5n.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5n.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6g.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6g.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c6g.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6g.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6g.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6g.large": {ENILimit: 3, IPv4Limit: 10}, - "c6g.medium": {ENILimit: 2, IPv4Limit: 4}, - "c6g.metal": {ENILimit: 15, IPv4Limit: 50}, - "c6g.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gd.large": {ENILimit: 3, IPv4Limit: 10}, - "c6gd.medium": {ENILimit: 2, IPv4Limit: 4}, - "c6gd.metal": {ENILimit: 15, IPv4Limit: 50}, - "c6gd.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6gn.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gn.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "c6gn.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "c6gn.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gn.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "c6gn.large": {ENILimit: 3, IPv4Limit: 10}, - "c6gn.medium": {ENILimit: 2, IPv4Limit: 4}, - "c6gn.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "cc2.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "cr1.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "d2.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "d2.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "d2.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "d2.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "d3.2xlarge": {ENILimit: 4, IPv4Limit: 5}, - "d3.4xlarge": {ENILimit: 4, IPv4Limit: 10}, - "d3.8xlarge": {ENILimit: 3, IPv4Limit: 20}, - "d3.xlarge": {ENILimit: 4, IPv4Limit: 3}, - "d3en.12xlarge": {ENILimit: 3, IPv4Limit: 30}, - "d3en.2xlarge": {ENILimit: 4, IPv4Limit: 5}, - "d3en.4xlarge": {ENILimit: 4, IPv4Limit: 10}, - "d3en.6xlarge": {ENILimit: 4, IPv4Limit: 15}, - "d3en.8xlarge": {ENILimit: 4, IPv4Limit: 20}, - "d3en.xlarge": {ENILimit: 4, IPv4Limit: 3}, - "f1.16xlarge": {ENILimit: 8, IPv4Limit: 50}, - "f1.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "f1.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g2.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "g2.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g3.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "g3.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g3.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g3s.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "g4ad.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g4ad.4xlarge": {ENILimit: 3, IPv4Limit: 10}, - "g4ad.8xlarge": {ENILimit: 4, IPv4Limit: 15}, - "g4dn.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "g4dn.16xlarge": {ENILimit: 4, IPv4Limit: 15}, - "g4dn.2xlarge": {ENILimit: 3, IPv4Limit: 10}, - "g4dn.4xlarge": {ENILimit: 3, IPv4Limit: 10}, - "g4dn.8xlarge": {ENILimit: 4, IPv4Limit: 15}, - "g4dn.metal": {ENILimit: 15, IPv4Limit: 50}, - "g4dn.xlarge": {ENILimit: 3, IPv4Limit: 10}, - "h1.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "h1.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "h1.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "h1.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "hs1.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i2.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i2.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i2.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i2.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i3.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "i3.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i3.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i3.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i3.large": {ENILimit: 3, IPv4Limit: 10}, - "i3.metal": {ENILimit: 15, IPv4Limit: 50}, - "i3.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i3en.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i3en.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "i3en.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i3en.3xlarge": {ENILimit: 4, IPv4Limit: 15}, - "i3en.6xlarge": {ENILimit: 8, IPv4Limit: 30}, - "i3en.large": {ENILimit: 3, IPv4Limit: 10}, - "i3en.metal": {ENILimit: 15, IPv4Limit: 50}, - "i3en.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "inf1.24xlarge": {ENILimit: 11, IPv4Limit: 30}, - "inf1.2xlarge": {ENILimit: 4, IPv4Limit: 10}, - "inf1.6xlarge": {ENILimit: 8, IPv4Limit: 30}, - "inf1.xlarge": {ENILimit: 4, IPv4Limit: 10}, - "m1.large": {ENILimit: 3, IPv4Limit: 10}, - "m1.medium": {ENILimit: 2, IPv4Limit: 6}, - "m1.small": {ENILimit: 2, IPv4Limit: 4}, - "m1.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m2.2xlarge": {ENILimit: 4, IPv4Limit: 30}, - "m2.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m2.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m3.2xlarge": {ENILimit: 4, IPv4Limit: 30}, - "m3.large": {ENILimit: 3, IPv4Limit: 10}, - "m3.medium": {ENILimit: 2, IPv4Limit: 6}, - "m3.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m4.10xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m4.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m4.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m4.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m4.large": {ENILimit: 2, IPv4Limit: 10}, - "m4.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5.large": {ENILimit: 3, IPv4Limit: 10}, - "m5.metal": {ENILimit: 15, IPv4Limit: 50}, - "m5.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5a.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5a.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5a.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5a.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5a.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5a.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5a.large": {ENILimit: 3, IPv4Limit: 10}, - "m5a.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5ad.large": {ENILimit: 3, IPv4Limit: 10}, - "m5ad.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5d.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5d.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5d.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5d.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5d.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5d.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5d.large": {ENILimit: 3, IPv4Limit: 10}, - "m5d.metal": {ENILimit: 15, IPv4Limit: 50}, - "m5d.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5dn.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5dn.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5dn.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5dn.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5dn.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5dn.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5dn.large": {ENILimit: 3, IPv4Limit: 10}, - "m5dn.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5n.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5n.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5n.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5n.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5n.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5n.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5n.large": {ENILimit: 3, IPv4Limit: 10}, - "m5n.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5zn.12xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m5zn.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m5zn.3xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5zn.6xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m5zn.large": {ENILimit: 3, IPv4Limit: 10}, - "m5zn.metal": {ENILimit: 15, IPv4Limit: 50}, - "m5zn.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m6g.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6g.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m6g.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m6g.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6g.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6g.large": {ENILimit: 3, IPv4Limit: 10}, - "m6g.medium": {ENILimit: 2, IPv4Limit: 4}, - "m6g.metal": {ENILimit: 15, IPv4Limit: 50}, - "m6g.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "m6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "m6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "m6gd.large": {ENILimit: 3, IPv4Limit: 10}, - "m6gd.medium": {ENILimit: 2, IPv4Limit: 4}, - "m6gd.metal": {ENILimit: 15, IPv4Limit: 50}, - "m6gd.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "mac1.metal": {ENILimit: 8, IPv4Limit: 30}, - "p2.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "p2.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "p2.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "p3.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "p3.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "p3.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "p3dn.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "p4d.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r3.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r3.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r3.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r3.large": {ENILimit: 3, IPv4Limit: 10}, - "r3.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r4.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r4.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r4.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r4.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r4.large": {ENILimit: 3, IPv4Limit: 10}, - "r4.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5.large": {ENILimit: 3, IPv4Limit: 10}, - "r5.metal": {ENILimit: 15, IPv4Limit: 50}, - "r5.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5a.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5a.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5a.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5a.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5a.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5a.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5a.large": {ENILimit: 3, IPv4Limit: 10}, - "r5a.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5ad.large": {ENILimit: 3, IPv4Limit: 10}, - "r5ad.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5b.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5b.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5b.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5b.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5b.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5b.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5b.large": {ENILimit: 3, IPv4Limit: 10}, - "r5b.metal": {ENILimit: 15, IPv4Limit: 50}, - "r5b.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5d.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5d.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5d.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5d.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5d.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5d.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5d.large": {ENILimit: 3, IPv4Limit: 10}, - "r5d.metal": {ENILimit: 15, IPv4Limit: 50}, - "r5d.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5dn.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5dn.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5dn.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5dn.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5dn.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5dn.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5dn.large": {ENILimit: 3, IPv4Limit: 10}, - "r5dn.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5n.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5n.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5n.24xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r5n.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r5n.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5n.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r5n.large": {ENILimit: 3, IPv4Limit: 10}, - "r5n.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r6g.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6g.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r6g.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r6g.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6g.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6g.large": {ENILimit: 3, IPv4Limit: 10}, - "r6g.medium": {ENILimit: 2, IPv4Limit: 4}, - "r6g.metal": {ENILimit: 15, IPv4Limit: 50}, - "r6g.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50}, - "r6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "r6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "r6gd.large": {ENILimit: 3, IPv4Limit: 10}, - "r6gd.medium": {ENILimit: 2, IPv4Limit: 4}, - "r6gd.metal": {ENILimit: 15, IPv4Limit: 50}, - "r6gd.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t1.micro": {ENILimit: 2, IPv4Limit: 2}, - "t2.2xlarge": {ENILimit: 3, IPv4Limit: 15}, - "t2.large": {ENILimit: 3, IPv4Limit: 12}, - "t2.medium": {ENILimit: 3, IPv4Limit: 6}, - "t2.micro": {ENILimit: 2, IPv4Limit: 2}, - "t2.nano": {ENILimit: 2, IPv4Limit: 2}, - "t2.small": {ENILimit: 3, IPv4Limit: 4}, - "t2.xlarge": {ENILimit: 3, IPv4Limit: 15}, - "t3.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t3.large": {ENILimit: 3, IPv4Limit: 12}, - "t3.medium": {ENILimit: 3, IPv4Limit: 6}, - "t3.micro": {ENILimit: 2, IPv4Limit: 2}, - "t3.nano": {ENILimit: 2, IPv4Limit: 2}, - "t3.small": {ENILimit: 3, IPv4Limit: 4}, - "t3.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t3a.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t3a.large": {ENILimit: 3, IPv4Limit: 12}, - "t3a.medium": {ENILimit: 3, IPv4Limit: 6}, - "t3a.micro": {ENILimit: 2, IPv4Limit: 2}, - "t3a.nano": {ENILimit: 2, IPv4Limit: 2}, - "t3a.small": {ENILimit: 2, IPv4Limit: 4}, - "t3a.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t4g.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "t4g.large": {ENILimit: 3, IPv4Limit: 12}, - "t4g.medium": {ENILimit: 3, IPv4Limit: 6}, - "t4g.micro": {ENILimit: 2, IPv4Limit: 2}, - "t4g.nano": {ENILimit: 2, IPv4Limit: 2}, - "t4g.small": {ENILimit: 3, IPv4Limit: 4}, - "t4g.xlarge": {ENILimit: 4, IPv4Limit: 15}, - "u-12tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "u-18tb1.metal": {ENILimit: 15, IPv4Limit: 50}, - "u-24tb1.metal": {ENILimit: 15, IPv4Limit: 50}, - "u-6tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "u-9tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "x1.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "x1.32xlarge": {ENILimit: 8, IPv4Limit: 30}, - "x1e.16xlarge": {ENILimit: 8, IPv4Limit: 30}, - "x1e.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "x1e.32xlarge": {ENILimit: 8, IPv4Limit: 30}, - "x1e.4xlarge": {ENILimit: 4, IPv4Limit: 15}, - "x1e.8xlarge": {ENILimit: 4, IPv4Limit: 15}, - "x1e.xlarge": {ENILimit: 3, IPv4Limit: 10}, - "z1d.12xlarge": {ENILimit: 15, IPv4Limit: 50}, - "z1d.2xlarge": {ENILimit: 4, IPv4Limit: 15}, - "z1d.3xlarge": {ENILimit: 8, IPv4Limit: 30}, - "z1d.6xlarge": {ENILimit: 8, IPv4Limit: 30}, - "z1d.large": {ENILimit: 3, IPv4Limit: 10}, - "z1d.metal": {ENILimit: 15, IPv4Limit: 50}, - "z1d.xlarge": {ENILimit: 4, IPv4Limit: 15}, + "a1.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "a1.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "a1.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "a1.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "a1.metal": {ENILimit: 8, IPv4Limit: 30, HypervisorType: ""}, + "a1.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c1.medium": {ENILimit: 2, IPv4Limit: 6, HypervisorType: "xen"}, + "c1.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "c3.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "c3.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "c3.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "c3.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "c3.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "c4.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "c4.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "c4.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "c4.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "c4.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "c5.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5.18xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5.9xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c5.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "c5.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5a.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5a.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5a.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5a.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5a.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5a.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5a.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c5a.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "c5a.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5ad.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c5ad.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "c5ad.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5d.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5d.18xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5d.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5d.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5d.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5d.9xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5d.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c5d.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "c5d.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5n.18xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c5n.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c5n.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5n.9xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c5n.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c5n.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "c5n.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6g.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6g.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c6g.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6g.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6g.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6g.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c6g.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "c6g.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "c6g.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gd.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c6gd.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "c6gd.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "c6gd.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6gn.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gn.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "c6gn.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "c6gn.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gn.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "c6gn.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "c6gn.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "c6gn.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "cc2.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "cr1.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "unkown"}, + "d2.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "d2.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "d2.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "d2.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "d3.2xlarge": {ENILimit: 4, IPv4Limit: 5, HypervisorType: "nitro"}, + "d3.4xlarge": {ENILimit: 4, IPv4Limit: 10, HypervisorType: "nitro"}, + "d3.8xlarge": {ENILimit: 3, IPv4Limit: 20, HypervisorType: "nitro"}, + "d3.xlarge": {ENILimit: 4, IPv4Limit: 3, HypervisorType: "nitro"}, + "d3en.12xlarge": {ENILimit: 3, IPv4Limit: 30, HypervisorType: "nitro"}, + "d3en.2xlarge": {ENILimit: 4, IPv4Limit: 5, HypervisorType: "nitro"}, + "d3en.4xlarge": {ENILimit: 4, IPv4Limit: 10, HypervisorType: "nitro"}, + "d3en.6xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "d3en.8xlarge": {ENILimit: 4, IPv4Limit: 20, HypervisorType: "nitro"}, + "d3en.xlarge": {ENILimit: 4, IPv4Limit: 3, HypervisorType: "nitro"}, + "f1.16xlarge": {ENILimit: 8, IPv4Limit: 50, HypervisorType: "xen"}, + "f1.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "f1.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "g2.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "g2.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "g3.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "xen"}, + "g3.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "g3.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "g3s.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "g4ad.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "g4ad.4xlarge": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "g4ad.8xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "g4dn.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "g4dn.16xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "g4dn.2xlarge": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "g4dn.4xlarge": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "g4dn.8xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "g4dn.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "g4dn.xlarge": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "h1.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "xen"}, + "h1.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "h1.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "h1.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "hs1.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "unkown"}, + "i2.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "i2.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "i2.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "i2.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "i3.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "xen"}, + "i3.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "i3.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "i3.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "i3.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "i3.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "i3.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "i3en.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "i3en.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "i3en.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "i3en.3xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "i3en.6xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "i3en.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "i3en.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "i3en.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "inf1.24xlarge": {ENILimit: 11, IPv4Limit: 30, HypervisorType: "nitro"}, + "inf1.2xlarge": {ENILimit: 4, IPv4Limit: 10, HypervisorType: "nitro"}, + "inf1.6xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "inf1.xlarge": {ENILimit: 4, IPv4Limit: 10, HypervisorType: "nitro"}, + "m1.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "m1.medium": {ENILimit: 2, IPv4Limit: 6, HypervisorType: "xen"}, + "m1.small": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "xen"}, + "m1.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "m2.2xlarge": {ENILimit: 4, IPv4Limit: 30, HypervisorType: "xen"}, + "m2.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "m2.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "m3.2xlarge": {ENILimit: 4, IPv4Limit: 30, HypervisorType: "xen"}, + "m3.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "m3.medium": {ENILimit: 2, IPv4Limit: 6, HypervisorType: "xen"}, + "m3.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "m4.10xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "m4.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "m4.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "m4.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "m4.large": {ENILimit: 2, IPv4Limit: 10, HypervisorType: "xen"}, + "m4.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "m5.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m5.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5a.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5a.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5a.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5a.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5a.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5a.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5a.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5a.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5ad.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5ad.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5d.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5d.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5d.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5d.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5d.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5d.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5d.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5d.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m5d.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5dn.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5dn.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5dn.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5dn.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5dn.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5dn.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5dn.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5dn.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m5dn.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5n.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5n.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5n.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5n.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5n.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5n.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5n.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5n.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m5n.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5zn.12xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m5zn.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m5zn.3xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5zn.6xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m5zn.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m5zn.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m5zn.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m6g.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6g.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m6g.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m6g.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6g.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6g.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m6g.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "m6g.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m6g.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "m6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "m6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "m6gd.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "m6gd.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "m6gd.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "m6gd.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "mac1.metal": {ENILimit: 8, IPv4Limit: 30, HypervisorType: ""}, + "p2.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "p2.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "p2.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "p3.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "p3.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "p3.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "p3dn.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "p4d.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "r3.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "r3.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "r3.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "r3.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "r3.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "r4.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "xen"}, + "r4.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "r4.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "r4.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "r4.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "r4.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "r5.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r5.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5a.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5a.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5a.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5a.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5a.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5a.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5a.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5a.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5ad.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5ad.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5ad.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5ad.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5ad.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5ad.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5ad.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5ad.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5b.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5b.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5b.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5b.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5b.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5b.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5b.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5b.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r5b.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5d.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5d.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5d.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5d.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5d.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5d.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5d.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5d.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r5d.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5dn.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5dn.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5dn.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5dn.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5dn.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5dn.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5dn.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5dn.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r5dn.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5n.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5n.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5n.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r5n.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r5n.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5n.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r5n.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r5n.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r5n.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r6g.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6g.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r6g.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r6g.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6g.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6g.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r6g.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "r6g.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r6g.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r6gd.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6gd.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "r6gd.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "r6gd.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6gd.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "r6gd.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "r6gd.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "r6gd.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "r6gd.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t1.micro": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "xen"}, + "t2.2xlarge": {ENILimit: 3, IPv4Limit: 15, HypervisorType: "xen"}, + "t2.large": {ENILimit: 3, IPv4Limit: 12, HypervisorType: "xen"}, + "t2.medium": {ENILimit: 3, IPv4Limit: 6, HypervisorType: "xen"}, + "t2.micro": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "xen"}, + "t2.nano": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "xen"}, + "t2.small": {ENILimit: 3, IPv4Limit: 4, HypervisorType: "xen"}, + "t2.xlarge": {ENILimit: 3, IPv4Limit: 15, HypervisorType: "xen"}, + "t3.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t3.large": {ENILimit: 3, IPv4Limit: 12, HypervisorType: "nitro"}, + "t3.medium": {ENILimit: 3, IPv4Limit: 6, HypervisorType: "nitro"}, + "t3.micro": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t3.nano": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t3.small": {ENILimit: 3, IPv4Limit: 4, HypervisorType: "nitro"}, + "t3.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t3a.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t3a.large": {ENILimit: 3, IPv4Limit: 12, HypervisorType: "nitro"}, + "t3a.medium": {ENILimit: 3, IPv4Limit: 6, HypervisorType: "nitro"}, + "t3a.micro": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t3a.nano": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t3a.small": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "t3a.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t4g.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "t4g.large": {ENILimit: 3, IPv4Limit: 12, HypervisorType: "nitro"}, + "t4g.medium": {ENILimit: 3, IPv4Limit: 6, HypervisorType: "nitro"}, + "t4g.micro": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t4g.nano": {ENILimit: 2, IPv4Limit: 2, HypervisorType: "nitro"}, + "t4g.small": {ENILimit: 3, IPv4Limit: 4, HypervisorType: "nitro"}, + "t4g.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "u-12tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "u-18tb1.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "u-24tb1.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "u-6tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "u-9tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "x1.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "x1.32xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "x1e.16xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "x1e.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "x1e.32xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "xen"}, + "x1e.4xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "x1e.8xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "xen"}, + "x1e.xlarge": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "xen"}, + "x2gd.12xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "x2gd.16xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "x2gd.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "x2gd.4xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "x2gd.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "x2gd.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "x2gd.medium": {ENILimit: 2, IPv4Limit: 4, HypervisorType: "nitro"}, + "x2gd.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "x2gd.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "z1d.12xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "nitro"}, + "z1d.2xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, + "z1d.3xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "z1d.6xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "nitro"}, + "z1d.large": {ENILimit: 3, IPv4Limit: 10, HypervisorType: "nitro"}, + "z1d.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: ""}, + "z1d.xlarge": {ENILimit: 4, IPv4Limit: 15, HypervisorType: "nitro"}, } diff --git a/pkg/ipamd/.vscode/settings.json b/pkg/ipamd/.vscode/settings.json new file mode 100644 index 0000000000..a460645592 --- /dev/null +++ b/pkg/ipamd/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "go.inferGopath": false +} \ No newline at end of file diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 962be14fde..78618dd4ac 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -15,6 +15,7 @@ package datastore import ( "fmt" + "net" "os" "sync" "time" @@ -144,11 +145,15 @@ type ENI struct { IsTrunk bool // IsEFA indicates whether this ENI is tagged as an EFA IsEFA bool + // IsPDEnabled indicates whether prefix delegation is enabled + IsPDEnabled bool // DeviceNumber is the device number of ENI (0 means the primary ENI) DeviceNumber int // IPv4Addresses shows whether each address is assigned, the key is IP address, which must // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") IPv4Addresses map[string]*AddressInfo + // Key is the prefix and value is prefix, used IPs under the prefix + IPv4Prefixes map[string]*ENIPrefix } // AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. @@ -156,6 +161,7 @@ type AddressInfo struct { IPAMKey IPAMKey Address string UnassignedTime time.Time + Prefix net.IP } func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) *AddressInfo { @@ -223,6 +229,7 @@ type PodIPInfo struct { type DataStore struct { total int assigned int + allocatedPrefix int eniPool ENIPool lock sync.Mutex log logger.Logger @@ -343,22 +350,59 @@ func (ds *DataStore) ReadBackingStore() error { defer ds.lock.Unlock() eniIPs := make(ENIPool) + eniPrefixes := make(ENIPool) + for _, eni := range ds.eniPool { for _, addr := range eni.IPv4Addresses { eniIPs[addr.Address] = eni } + for _, prefix := range eni.IPv4Prefixes { + eniPrefixes[prefix.Prefix.IP.String()] = eni + } } for _, allocation := range data.Allocations { eni := eniIPs[allocation.IPv4] if eni == nil { - ds.log.Infof("datastore: Sandbox %s uses unknown IPv4 %s - presuming stale/dead", allocation.IPAMKey, allocation.IPv4) - continue - } + //Check if this IP belongs to prefix, this can happen when knob is toggled from enable to disable with pods assigned + ds.log.Infof("Got IP to recover %s\n", allocation.IPv4) + ipv4Prefix := getPrefixFromIPv4Addr(allocation.IPv4) + ds.log.Infof("Retrieved prefix %s", ipv4Prefix.String()) + + eni := eniPrefixes[ipv4Prefix.String()] + if eni == nil { + ds.log.Infof("datastore: Sandbox %s uses unknown IPv4 %s - presuming stale/dead", allocation.IPAMKey, allocation.IPv4) + continue + } + eniPrefixDB := ds.eniPool[eni.ID].IPv4Prefixes[ipv4Prefix.String()] + + eniPrefixDB.UsedIPs++ + eniPrefixDB.FreeIps-- + + err := eniPrefixDB.AllocatedIPs.restorePrefixIP(allocation.IPv4) + if err != nil { + ds.log.Errorf("Failed to ADD PD IP %s on ENI %s", allocation.IPv4, eni.ID) + return err + } + + err = ds.AddPrefixIPv4AddressToStore(eni.ID, allocation.IPv4) + if err != nil { + ds.log.Errorf("Failed to ADD PD IP %s on ENI %s", allocation.IPv4, eni.ID) + return err + } - addr := eni.IPv4Addresses[allocation.IPv4] - ds.assignPodIPv4AddressUnsafe(allocation.IPAMKey, eni, addr) - ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) + addr := eni.IPv4Addresses[allocation.IPv4] + ds.assignPodIPv4AddressUnsafe(allocation.IPAMKey, eni, addr) + + addr.Prefix = ipv4Prefix + + ds.log.Debugf("Recovered PD prefix %s ", ipv4Prefix.String()) + ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) + } else { + addr := eni.IPv4Addresses[allocation.IPv4] + ds.assignPodIPv4AddressUnsafe(allocation.IPAMKey, eni, addr) + ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) + } } if ds.CheckpointMigrationPhase == 1 { @@ -399,11 +443,11 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { } // AddENI add ENI to data store -func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, isEFA bool) error { +func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, isEFA, isPDEnabled bool) error { ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("DataStore Add an ENI %s", eniID) + ds.log.Debugf("DataStore Add an ENI %s and is PD enabled %d", eniID, isPDEnabled) _, ok := ds.eniPool[eniID] if ok { @@ -414,9 +458,12 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, IsPrimary: isPrimary, IsTrunk: isTrunk, IsEFA: isEFA, + IsPDEnabled: isPDEnabled, ID: eniID, DeviceNumber: deviceNumber, - IPv4Addresses: make(map[string]*AddressInfo)} + IPv4Addresses: make(map[string]*AddressInfo), + IPv4Prefixes: make(map[string]*ENIPrefix)} + enis.Set(float64(len(ds.eniPool))) return nil } @@ -426,14 +473,17 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { ds.lock.Lock() defer ds.lock.Unlock() + ds.log.Infof("Adding %s to DS for %s", ipv4, eniID) curENI, ok := ds.eniPool[eniID] if !ok { + ds.log.Infof("unkown ENI") return errors.New("add ENI's IP to datastore: unknown ENI") } // Already there _, ok = curENI.IPv4Addresses[ipv4] if ok { + ds.log.Infof("IP already in DS") return errors.New(IPAlreadyInStoreError) } @@ -446,6 +496,63 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { return nil } +// AddPrefixIPv4AddressToStore add an IP of an ENI to data store +func (ds *DataStore) AddPrefixIPv4AddressToStore(eniID string, ipv4 string) error { + + ds.log.Infof("Adding %s to DS for %s", ipv4, eniID) + curENI, ok := ds.eniPool[eniID] + if !ok { + ds.log.Infof("unkown ENI") + return errors.New("add ENI's IP to datastore: unknown ENI") + } + + // Already there + _, ok = curENI.IPv4Addresses[ipv4] + if ok { + ds.log.Infof("IP already in DS") + return errors.New(IPAlreadyInStoreError) + } + + // Prometheus gauge + totalIPs.Set(float64(ds.total)) + + curENI.IPv4Addresses[ipv4] = &AddressInfo{Address: ipv4} + ds.log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) + return nil +} + +// AddIPv4PrefixToStore add an IP of an ENI to data store +func (ds *DataStore) AddIPv4PrefixToStore(eniID string, ipv4Prefix string) error { + ds.lock.Lock() + defer ds.lock.Unlock() + + curENI, ok := ds.eniPool[eniID] + if !ok { + return errors.New("add ENI's IP to datastore: unknown ENI") + } + //split IP here + ip, ipnet, err := net.ParseCIDR(ipv4Prefix) + if err != nil { + return err + } + prefix := net.IPNet{IP: ip, Mask: ipnet.Mask} + + // Already there + _, ok = curENI.IPv4Prefixes[prefix.IP.String()] + if ok { + return errors.New(IPAlreadyInStoreError) + } + _, numIPsPerPrefix, _ := GetPrefixDelegationDefaults() + ds.total = ds.total + numIPsPerPrefix + ds.allocatedPrefix++ + // Prometheus gauge + totalIPs.Set(float64(ds.total)) + + curENI.IPv4Prefixes[prefix.IP.String()] = &ENIPrefix{Prefix: prefix, AllocatedIPs: NewPrefixStore(numIPsPerPrefix), UsedIPs: 0, FreeIps: numIPsPerPrefix} + ds.log.Infof("Added ENI(%s)'s IPv4 Prefix %s to datastore", eniID, prefix.IP.String()) + return nil +} + // DelIPv4AddressFromStore delete an IP of ENI from datastore func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bool) error { ds.lock.Lock() @@ -484,6 +591,62 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo return nil } +// DelIPv4AddressFromStore delete an IP of ENI from datastore +func (ds *DataStore) DelIPv4PrefixFromStore(eniID string, ipv4Prefix string, force bool) error { + ds.lock.Lock() + defer ds.lock.Unlock() + + curENI, ok := ds.eniPool[eniID] + if !ok { + return errors.New(UnknownENIError) + } + + //split IP here + ip, ipnet, err := net.ParseCIDR(ipv4Prefix) + if err != nil { + return err + } + + ipv4 := ip.String() + prefixlen := ipnet.Mask + ds.log.Debugf("Adding IP %s and prefix len %s", ipv4, prefixlen) + + ipPrefix, ok := curENI.IPv4Prefixes[ipv4] + if !ok { + return errors.New(UnknownIPError) + } + + if ipPrefix.UsedIPs != 0 { + if !force { + return errors.New(IPInUseError) + } + ds.log.Warnf("Force deleting assigned Prefix %s on eni %s", ipv4, eniID) + forceRemovedIPs.Inc() + //Cleanup datastore /32 ips + prefixDB := ipPrefix.AllocatedIPs + + for strPrivateIPv4, _ := range prefixDB.UsedIPs { + addr := curENI.IPv4Addresses[strPrivateIPv4] + ds.unassignPodIPv4AddressUnsafe(addr) + } + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Unable to update backing store: %v", err) + // Continuing because 'force' + } + } + + _, numIPsPerPrefix, _ := GetPrefixDelegationDefaults() + ds.total = ds.total - numIPsPerPrefix + ds.allocatedPrefix-- + // Prometheus gauge + totalIPs.Set(float64(ds.total)) + + delete(curENI.IPv4Prefixes, ipv4) + + ds.log.Infof("Deleted ENI(%s)'s Prefix %s from datastore", eniID, ipv4) + return nil +} + // AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, deviceNumber int, err error) { @@ -491,23 +654,64 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, defer ds.lock.Unlock() ds.log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned) + if eni, addr := ds.eniPool.FindAddressForSandbox(ipamKey); addr != nil { ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for sandbox %s", ipamKey) return addr.Address, eni.DeviceNumber, nil } for _, eni := range ds.eniPool { - for _, addr := range eni.IPv4Addresses { - if !addr.Assigned() && !addr.inCoolingPeriod() { - ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) - if err := ds.writeBackingStoreUnsafe(); err != nil { - ds.log.Warnf("Failed to update backing store: %v", err) - // Important! Unwind assignment - ds.unassignPodIPv4AddressUnsafe(addr) - return "", -1, err + ds.log.Debugf("Checking for eni %v", eni) + if !eni.IsPDEnabled { + for _, addr := range eni.IPv4Addresses { + if !addr.Assigned() && !addr.inCoolingPeriod() { + ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Failed to update backing store: %v", err) + // Important! Unwind assignment + ds.unassignPodIPv4AddressUnsafe(addr) + return "", -1, err + } + + return addr.Address, eni.DeviceNumber, nil + } + } + } else { + ds.log.Debugf("PD is enabled.") + for _, prefix := range eni.IPv4Prefixes { + ds.log.Debugf("Found a prefix %s", prefix) + if prefix.FreeIps > 0 { + strPrivateIPv4, err := getFreeIPv4AddrfromPrefix(prefix) + if err != nil { + ds.log.Errorf("Unable to get IP address from prefix: %v", err) + return "", -1, err + } + ds.log.Debugf("New IP - %s", strPrivateIPv4) + // Try to add the IP + + err = ds.AddPrefixIPv4AddressToStore(eni.ID, strPrivateIPv4) + if err != nil { + ds.log.Errorf("Failed to ADD PD IP %s on ENI %s", strPrivateIPv4, eni) + return "", -1, err + } + + //Adding here to eni DB, need to be removed while deleteing. + addr := eni.IPv4Addresses[strPrivateIPv4] + ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + + addr.Prefix = prefix.Prefix.IP + + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Failed to update backing store: %v", err) + // Important! Unwind assignment + ds.unassignPodIPv4AddressUnsafe(addr) + deleteIPv4AddrfromPrefix(prefix, addr) + //Remove the IP from eni DB + delete(eni.IPv4Addresses, addr.Address) + return "", -1, err + } + return addr.Address, eni.DeviceNumber, nil } - - return addr.Address, eni.DeviceNumber, nil } } ds.log.Debugf("AssignPodIPv4Address: ENI %s does not have available addresses", eni.ID) @@ -547,8 +751,8 @@ func (ds *DataStore) unassignPodIPv4AddressUnsafe(addr *AddressInfo) { } // GetStats returns total number of IP addresses and number of assigned IP addresses -func (ds *DataStore) GetStats() (int, int) { - return ds.total, ds.assigned +func (ds *DataStore) GetStats() (int, int, int) { + return ds.total, ds.assigned, ds.allocatedPrefix } // GetTrunkENI returns the trunk ENI ID or an empty string @@ -600,7 +804,23 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) return otherIPs < minimumIPTarget } -func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI { +// IsRequiredForWarmPrefixTarget determines if this ENI is necessary to fulfill whatever WARM_PREFIX_TARGET is +// set to. +func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *ENI) bool { + freePrefixes := 0 + for _, other := range ds.eniPool { + if other.ID != eni.ID { + for _, otherPrefixes := range other.IPv4Prefixes { + if otherPrefixes.UsedIPs == 0 { + freePrefixes++ + } + } + } + } + return freePrefixes < warmPrefixTarget +} + +func (ds *DataStore) getDeletableENI(warmIPTarget, minimumIPTarget, warmPrefixTarget int) *ENI { for _, eni := range ds.eniPool { if eni.IsPrimary { ds.log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID) @@ -622,12 +842,12 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI continue } - if warmIPTarget != 0 && ds.isRequiredForWarmIPTarget(warmIPTarget, eni) { + if !eni.IsPDEnabled && warmIPTarget != 0 && ds.isRequiredForWarmIPTarget(warmIPTarget, eni) { ds.log.Debugf("ENI %s cannot be deleted because it is required for WARM_IP_TARGET: %d", eni.ID, warmIPTarget) continue } - if minimumIPTarget != 0 && ds.isRequiredForMinimumIPTarget(minimumIPTarget, eni) { + if !eni.IsPDEnabled && minimumIPTarget != 0 && ds.isRequiredForMinimumIPTarget(minimumIPTarget, eni) { ds.log.Debugf("ENI %s cannot be deleted because it is required for MINIMUM_IP_TARGET: %d", eni.ID, minimumIPTarget) continue } @@ -642,6 +862,11 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI continue } + if eni.IsPDEnabled && warmPrefixTarget != 0 && ds.isRequiredForWarmPrefixTarget(warmPrefixTarget, eni) { + ds.log.Debugf("ENI %s cannot be deleted because it is required for WARM_PREFIX_TARGET: %d", eni.ID, warmPrefixTarget) + continue + } + ds.log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID) return eni } @@ -672,16 +897,19 @@ func (e *ENI) hasPods() bool { func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENI { ds.lock.Lock() defer ds.lock.Unlock() - for _, eni := range ds.eniPool { if skipPrimary && eni.IsPrimary { ds.log.Debugf("Skip the primary ENI for need IP check") continue } - if len(eni.IPv4Addresses) < maxIPperENI { + if !eni.IsPDEnabled && len(eni.IPv4Addresses) < maxIPperENI { ds.log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", eni.ID, len(eni.IPv4Addresses), maxIPperENI) return eni + } else if eni.IsPDEnabled && len(eni.IPv4Prefixes) < maxIPperENI { + ds.log.Debugf("Found ENI %s that has less than the maximum number of v4 Prefixes allocated: cur=%d, max=%d", + eni.ID, len(eni.IPv4Prefixes), maxIPperENI) + return eni } } return nil @@ -690,20 +918,28 @@ func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENI { // RemoveUnusedENIFromStore removes a deletable ENI from the data store. // It returns the name of the ENI which has been removed from the data store and needs to be deleted, // or empty string if no ENI could be removed. -func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int, minimumIPTarget int) string { +func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget, minimumIPTarget, warmPrefixTarget int) string { ds.lock.Lock() defer ds.lock.Unlock() - deletableENI := ds.getDeletableENI(warmIPTarget, minimumIPTarget) + deletableENI := ds.getDeletableENI(warmIPTarget, minimumIPTarget, warmPrefixTarget) if deletableENI == nil { return "" } removableENI := deletableENI.ID - eniIPCount := len(ds.eniPool[removableENI].IPv4Addresses) - ds.total -= eniIPCount - ds.log.Infof("RemoveUnusedENIFromStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", - removableENI, eniIPCount, ds.total, ds.assigned) + if !ds.eniPool[removableENI].IsPDEnabled { + eniIPCount := len(ds.eniPool[removableENI].IPv4Addresses) + ds.total -= eniIPCount + ds.log.Infof("RemoveUnusedENIFromStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", + removableENI, eniIPCount, ds.total, ds.assigned) + } else { + _, numIPsPerPrefix, _ := GetPrefixDelegationDefaults() + ds.total = ds.total - (len(ds.eniPool[removableENI].IPv4Prefixes) * numIPsPerPrefix) + ds.allocatedPrefix -= len(ds.eniPool[removableENI].IPv4Prefixes) + ds.log.Infof("RemoveUnusedENIFromStore %s: Prefix address pool stats: free %d addresses, total: %d, assigned: %d total prefixes: %d", + removableENI, len(ds.eniPool[removableENI].IPv4Prefixes), ds.total, ds.assigned, ds.allocatedPrefix) + } delete(ds.eniPool, removableENI) // Prometheus update @@ -743,9 +979,17 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { } } - ds.total -= len(eni.IPv4Addresses) - ds.log.Infof("RemoveENIFromDataStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", - eniID, len(eni.IPv4Addresses), ds.total, ds.assigned) + if !eni.IsPDEnabled { + ds.total -= len(eni.IPv4Addresses) + ds.log.Infof("RemoveENIFromDataStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", + eniID, len(eni.IPv4Addresses), ds.total, ds.assigned) + } else { + _, numIPsPerPrefix, _ := GetPrefixDelegationDefaults() + ds.total = ds.total - (len(eni.IPv4Prefixes) * numIPsPerPrefix) + ds.allocatedPrefix -= len(eni.IPv4Prefixes) + ds.log.Infof("RemoveENIFromDataStore %s: Prefix address pool stats: free %d addresses, total: %d, assigned: %d total prefixes: %d", + eniID, len(eni.IPv4Prefixes), ds.total, ds.assigned, ds.allocatedPrefix) + } delete(ds.eniPool, eniID) // Prometheus gauge @@ -755,14 +999,14 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { // UnassignPodIPv4Address a) find out the IP address based on PodName and PodNameSpace // b) mark IP address as unassigned c) returns IP address, ENI's device number, error -func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (ip string, deviceNumber int, err error) { +func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, isSecondaryIP bool, err error) { ds.lock.Lock() defer ds.lock.Unlock() ds.log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey) eni, addr := ds.eniPool.FindAddressForSandbox(ipamKey) - + isSecondaryIP = false if addr == nil { // This `if` block should be removed when the CRI // migration code is finally removed. Leaving a @@ -781,20 +1025,36 @@ func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (ip string, deviceN if addr == nil { ds.log.Warnf("UnassignPodIPv4Address: Failed to find sandbox %s", ipamKey) - return "", 0, ErrUnknownPod + return nil, "", 0, false, ErrUnknownPod } ds.unassignPodIPv4AddressUnsafe(addr) if err := ds.writeBackingStoreUnsafe(); err != nil { // Unwind un-assignment ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) - return "", 0, err + return nil, "", 0, false, err } addr.UnassignedTime = time.Now() + retrievedPrefix := addr.Prefix + if eni.IsPDEnabled && retrievedPrefix == nil { + ds.log.Infof("Prefix delegation is enabled and the IP is from secondary pool hence no need to update prefix pool") + ds.total-- + isSecondaryIP = true + } else if retrievedPrefix != nil { + ds.log.Infof("Retrieved Prefix %s", retrievedPrefix.String()) + //Regular pod deletes for PD enabled and if pd is disabled but prefix is populated i.e, knob disable scenario + deleteIPv4AddrfromPrefix(eni.IPv4Prefixes[retrievedPrefix.String()], addr) + //Remove the IP from eni DB + delete(eni.IPv4Addresses, addr.Address) + if !eni.IsPDEnabled { + ds.log.Infof("Prefix delegation is disabled but IP belongs to prefix pool") + isSecondaryIP = true + } + } ds.log.Infof("UnassignPodIPv4Address: sandbox %s's ipAddr %s, DeviceNumber %d", ipamKey, addr.Address, eni.DeviceNumber) - return addr.Address, eni.DeviceNumber, nil + return eni, addr.Address, eni.DeviceNumber, isSecondaryIP, nil } // AllocatedIPs returns a recent snapshot of allocated sandbox<->IPs. @@ -841,6 +1101,29 @@ func (ds *DataStore) FreeableIPs(eniID string) []string { return freeable } +// FreeablePrefixes returns a list of unused and potentially freeable IPs. +// Note result may already be stale by the time you look at it. +func (ds *DataStore) FreeablePrefixes(eniID string) []string { + ds.lock.Lock() + defer ds.lock.Unlock() + + eni := ds.eniPool[eniID] + if eni == nil { + // Can't free any IPs from an ENI we don't know about... + return []string{} + } + ds.log.Debugf("In freeable prefix for eni %s", eniID) + + freeable := make([]string, 0, len(eni.IPv4Prefixes)) + for _, prefix := range eni.IPv4Prefixes { + if prefix.UsedIPs == 0 { + freeable = append(freeable, prefix.Prefix.String()) + } + } + + return freeable +} + // GetENIInfos provides ENI and IP information about the datastore func (ds *DataStore) GetENIInfos() *ENIInfos { ds.lock.Lock() @@ -883,8 +1166,58 @@ func (ds *DataStore) GetENIIPs(eniID string) ([]string, error) { } var ipPool = make([]string, 0, len(eni.IPv4Addresses)) - for ip := range eni.IPv4Addresses { + for ip, addr := range eni.IPv4Addresses { + // /32 IPs from SIP or PD pool will be in eni.IPv4Addresses + // hence this has to be checked, IP belonging to a prefix can be + // a possibility because of PD enable/disable knob toggle + if !eni.IsPDEnabled && addr.Prefix != nil { + ds.log.Debugf("IP %s belongs to prefix pool so do not account", ip) + continue + } ipPool = append(ipPool, ip) } return ipPool, nil } + +// GetENIPrefixes returns the known (allocated & unallocated) ENI Prefixed. +func (ds *DataStore) GetENIPrefixes(eniID string) ([]string, error) { + ds.lock.Lock() + defer ds.lock.Unlock() + + eni, ok := ds.eniPool[eniID] + if !ok { + return nil, errors.New(UnknownENIError) + } + + var ipPool = make([]string, 0, len(eni.IPv4Prefixes)) + for _, prefixData := range eni.IPv4Prefixes { + ipPool = append(ipPool, prefixData.Prefix.String()) + } + return ipPool, nil +} + +func (ds *DataStore) GetFreePrefixes() int { + ds.lock.Lock() + defer ds.lock.Unlock() + + freePrefixes := 0 + for _, other := range ds.eniPool { + for _, otherPrefixes := range other.IPv4Prefixes { + if otherPrefixes.UsedIPs == 0 { + freePrefixes++ + } + } + + } + return freePrefixes +} + +func (ds *DataStore) CleanupDataStore(eniID string, ipv4 string, force bool, enableIpv4PrefixDelegation bool) error { + var err error + if !enableIpv4PrefixDelegation { + err = ds.DelIPv4AddressFromStore(eniID, ipv4, force) + } else { + err = ds.DelIPv4PrefixFromStore(eniID, ipv4, force) + } + return err +} diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 9200b47cfb..3595828ab5 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -28,18 +28,18 @@ var logConfig = logger.Configuration{ LogLocation: "stdout", } -var log = logger.New(&logConfig) +var Testlog = logger.New(&logConfig) func TestAddENI(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) + ds := NewDataStore(Testlog, NullCheckpoint{}) - err := ds.AddENI("eni-1", 1, true, false, false) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-1", 1, true, false, false) + err = ds.AddENI("eni-1", 1, true, false, false, false) assert.Error(t, err) - err = ds.AddENI("eni-2", 2, false, false, false) + err = ds.AddENI("eni-2", 2, false, false, false, false) assert.NoError(t, err) assert.Equal(t, len(ds.eniPool), 2) @@ -49,15 +49,15 @@ func TestAddENI(t *testing.T) { } func TestDeleteENI(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) + ds := NewDataStore(Testlog, NullCheckpoint{}) - err := ds.AddENI("eni-1", 1, true, false, false) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-2", 2, false, false, false) + err = ds.AddENI("eni-2", 2, false, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-3", 3, false, false, false) + err = ds.AddENI("eni-3", 3, false, false, false, false) assert.NoError(t, err) eniInfos := ds.GetENIInfos() @@ -92,12 +92,12 @@ func TestDeleteENI(t *testing.T) { } func TestAddENIIPv4Address(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) + ds := NewDataStore(Testlog, NullCheckpoint{}) - err := ds.AddENI("eni-1", 1, true, false, false) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-2", 2, false, false, false) + err = ds.AddENI("eni-2", 2, false, false, false, false) assert.NoError(t, err) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") @@ -130,12 +130,12 @@ func TestAddENIIPv4Address(t *testing.T) { } func TestGetENIIPs(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) + ds := NewDataStore(Testlog, NullCheckpoint{}) - err := ds.AddENI("eni-1", 1, true, false, false) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-2", 2, false, false, false) + err = ds.AddENI("eni-2", 2, false, false, false, false) assert.NoError(t, err) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") @@ -163,8 +163,8 @@ func TestGetENIIPs(t *testing.T) { } func TestDelENIIPv4Address(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) - err := ds.AddENI("eni-1", 1, true, false, false) + ds := NewDataStore(Testlog, NullCheckpoint{}) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") @@ -215,12 +215,12 @@ func TestDelENIIPv4Address(t *testing.T) { func TestPodIPv4Address(t *testing.T) { checkpoint := NewTestCheckpoint(struct{}{}) - ds := NewDataStore(log, checkpoint) + ds := NewDataStore(Testlog, checkpoint) - err := ds.AddENI("eni-1", 1, true, false, false) + err := ds.AddENI("eni-1", 1, true, false, false, false) assert.NoError(t, err) - err = ds.AddENI("eni-2", 2, false, false, false) + err = ds.AddENI("eni-2", 2, false, false, false, false) assert.NoError(t, err) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") @@ -301,10 +301,10 @@ func TestPodIPv4Address(t *testing.T) { _, _, err = ds.AssignPodIPv4Address(key4) assert.Error(t, err) // Unassign unknown Pod - _, _, err = ds.UnassignPodIPv4Address(key4) + _, _, _, _, err = ds.UnassignPodIPv4Address(key4) assert.Error(t, err) - _, deviceNum, err := ds.UnassignPodIPv4Address(key2) + _, _, deviceNum, _, err := ds.UnassignPodIPv4Address(key2) assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 2) @@ -315,14 +315,15 @@ func TestPodIPv4Address(t *testing.T) { noWarmIPTarget := 0 noMinimumIPTarget := 0 + noWarmPrefixTarget := 0 // Should not be able to free this ENI - eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget) + eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) assert.True(t, eni == "") ds.eniPool["eni-2"].createTime = time.Time{} ds.eniPool["eni-2"].IPv4Addresses["1.1.2.2"].UnassignedTime = time.Time{} - eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget) + eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) assert.Equal(t, eni, "eni-2") assert.Equal(t, ds.total, 2) @@ -330,11 +331,11 @@ func TestPodIPv4Address(t *testing.T) { } func TestWarmENIInteractions(t *testing.T) { - ds := NewDataStore(log, NullCheckpoint{}) + ds := NewDataStore(Testlog, NullCheckpoint{}) - _ = ds.AddENI("eni-1", 1, true, false, false) - _ = ds.AddENI("eni-2", 2, false, false, false) - _ = ds.AddENI("eni-3", 3, false, false, false) + _ = ds.AddENI("eni-1", 1, true, false, false, false) + _ = ds.AddENI("eni-2", 2, false, false, false, false) + _ = ds.AddENI("eni-3", 3, false, false, false, false) _ = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") key1 := IPAMKey{"net0", "sandbox-1", "eth0"} @@ -357,33 +358,33 @@ func TestWarmENIInteractions(t *testing.T) { // We have three ENIs, 5 IPs and two pods on ENI 1. Each ENI can handle two pods. // We should not be able to remove any ENIs if either warmIPTarget >= 3 or minimumWarmIPTarget >= 5 - eni := ds.RemoveUnusedENIFromStore(3, 1) + eni := ds.RemoveUnusedENIFromStore(3, 1, 0) assert.Equal(t, "", eni) // Should not be able to free this ENI because we want at least 5 IPs, which requires at least three ENIs - eni = ds.RemoveUnusedENIFromStore(1, 5) + eni = ds.RemoveUnusedENIFromStore(1, 5, 0) assert.Equal(t, "", eni) // Should be able to free an ENI because both warmIPTarget and minimumWarmIPTarget are both effectively 4 - removedEni := ds.RemoveUnusedENIFromStore(2, 4) + removedEni := ds.RemoveUnusedENIFromStore(2, 4, 0) assert.Contains(t, []string{"eni-2", "eni-3"}, removedEni) // Should not be able to free an ENI because minimumWarmIPTarget requires at least two ENIs and no warm IP target - eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, 3) + eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, 3, 0) assert.Equal(t, "", eni) // Should be able to free an ENI because one ENI can provide a minimum count of 2 IPs - secondRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2) + secondRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2, 0) assert.Contains(t, []string{"eni-2", "eni-3"}, secondRemovedEni) assert.NotEqual(t, removedEni, secondRemovedEni, "The two removed ENIs should not be the same ENI.") - _ = ds.AddENI("eni-4", 3, false, true, false) - _ = ds.AddENI("eni-5", 3, false, false, true) + _ = ds.AddENI("eni-4", 3, false, true, false, false) + _ = ds.AddENI("eni-5", 3, false, false, true, false) _ = ds.AddIPv4AddressToStore("eni-4", "1.1.4.1") _ = ds.AddIPv4AddressToStore("eni-5", "1.1.5.1") ds.eniPool["eni-4"].createTime = time.Time{} ds.eniPool["eni-5"].createTime = time.Time{} - thirdRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2) + thirdRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2, 0) // None of the others can be removed... assert.Equal(t, "", thirdRemovedEni) assert.Equal(t, 3, ds.GetENIs()) diff --git a/pkg/ipamd/datastore/prefix_store.go b/pkg/ipamd/datastore/prefix_store.go new file mode 100644 index 0000000000..95ac2742a8 --- /dev/null +++ b/pkg/ipamd/datastore/prefix_store.go @@ -0,0 +1,153 @@ +package datastore + +import ( + "net" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "github.com/pkg/errors" +) + +const ( + //"/32" IPv4 prefix + ipv4DefaultPrefixSize = 32 +) + +var log = logger.Get() + +// PrefixIPsStore will hold the size of the prefix Eg: 16 for /28 prefix. In future if prefix sizes are customizable +// UsedIPs will be a map of +type PrefixIPsStore struct { + UsedIPs map[string]time.Time + IPsPerPrefix int +} + +// ENIPrefix will have the prefixes allocated for an ENI and the list of allocated IPs under +// the prefix +type ENIPrefix struct { + Prefix net.IPNet + AllocatedIPs PrefixIPsStore + UsedIPs int + FreeIps int +} + +//New - allocate a PrefixIPsStore per prefix of size "prefixsize" +func NewPrefixStore(prefixsize int) PrefixIPsStore { + return PrefixIPsStore{IPsPerPrefix: prefixsize, UsedIPs: make(map[string]time.Time)} +} + +// Size returns the number of IPs per prefix +func (prefix PrefixIPsStore) getPrefixSize() int { + return prefix.IPsPerPrefix +} + +func isOutofCoolingPeriod(cooldown time.Time) bool { + return (!(time.Since(cooldown) <= addressCoolingPeriod)) +} + +func getNextIP(ip net.IP) { + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } +} + +func (prefix PrefixIPsStore) getUnusedIP(eniPrefix string) (string, error) { + //Check if there is any IP out of cooldown + var cachedIP string + for ipv4, cooldown := range prefix.UsedIPs { + //Currently we have 2 DB's, IP allocated from the prefixDB is added to + //IP pool so instead of checking if the IP is assigned from IPAMKey [ipAddr.Assigned()] considering an + //ip with 0 cooldown time as assigned. Will fix this up as part of merging the 2 DBs. + if isOutofCoolingPeriod(cooldown) && !cooldown.IsZero() { + //if the IP is out of cooldown and not assigned then cache the first available IP + //continue cleaning up the DB, this is to avoid stale entries and a new thread :) + if cachedIP == "" { + cachedIP = ipv4 + } + delete(prefix.UsedIPs, ipv4) + } + } + if cachedIP != "" { + prefix.UsedIPs[cachedIP] = time.Time{} + return cachedIP, nil + } + + //If not in cooldown then generate next IP + ip, ipnet, err := net.ParseCIDR(eniPrefix) + if err != nil { + return "", err + } + + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); getNextIP(ip) { + strPrivateIPv4 := ip.String() + if _, ok := prefix.UsedIPs[strPrivateIPv4]; ok { + continue + } + log.Debugf("Found a free IP not in DB - %s", strPrivateIPv4) + return strPrivateIPv4, nil + } + + return "", errors.New("No free IP in the prefix store") +} + +func (prefix PrefixIPsStore) restorePrefixIP(recoveredIP string) error { + // Already there + _, ok := prefix.UsedIPs[recoveredIP] + if ok { + log.Debugf("IP already in DS") + return errors.New(IPAlreadyInStoreError) + } + prefix.UsedIPs[recoveredIP] = time.Time{} + log.Debugf("Recovered IP in prefix store %s", recoveredIP) + return nil + +} + +//Given prefix, this function returns a free IP in the prefix +func getFreeIPv4AddrfromPrefix(prefix *ENIPrefix) (string, error) { + if prefix == nil { + log.Errorf("Prefix datastore not initialized") + return "", errors.New("Prefix datastore not initialized") + } + strPrivateIPv4, err := prefix.AllocatedIPs.getUnusedIP(prefix.Prefix.String()) + if err != nil { + log.Debugf("Get free IP from prefix failed %v", err) + return "", err + } + prefix.AllocatedIPs.UsedIPs[strPrivateIPv4] = time.Time{} + prefix.FreeIps-- + prefix.UsedIPs++ + log.Debugf("Returning Free IP %s", strPrivateIPv4) + return strPrivateIPv4, nil +} + +//This function sets the cooldown for the IP freed from the prefix +func deleteIPv4AddrfromPrefix(prefix *ENIPrefix, addr *AddressInfo) { + prefix.AllocatedIPs.UsedIPs[addr.Address] = addr.UnassignedTime + log.Debugf("Setting cooldown for IP address %s at time %v", addr.Address, addr.UnassignedTime) + + prefix.FreeIps++ + prefix.UsedIPs-- +} + +//Given a IPv4 address, this will compute the associated prefix +func getPrefixFromIPv4Addr(IPaddr string) net.IP { + _, _, supportedPrefixLen := GetPrefixDelegationDefaults() + ipv4Prefix := net.ParseIP(IPaddr) + ipv4PrefixMask := net.CIDRMask(supportedPrefixLen, ipv4DefaultPrefixSize) + ipv4Prefix = ipv4Prefix.To4() + ipv4Prefix = ipv4Prefix.Mask(ipv4PrefixMask) + return ipv4Prefix +} + +//Function to return PD defaults supported by VPC +func GetPrefixDelegationDefaults() (int, int, int) { + numPrefixesPerENI := 1 + numIPsPerPrefix := 16 + supportedPrefixLen := 28 + + return numPrefixesPerENI, numIPsPerPrefix, supportedPrefixLen +} diff --git a/pkg/ipamd/datastore/prefix_store_test.go b/pkg/ipamd/datastore/prefix_store_test.go new file mode 100644 index 0000000000..0ca5296a11 --- /dev/null +++ b/pkg/ipamd/datastore/prefix_store_test.go @@ -0,0 +1,489 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package datastore + +import ( + "errors" + "testing" + "time" + + //"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "github.com/stretchr/testify/assert" +) + +/* +var logConfig = logger.Configuration{ + LogLevel: "Debug", + LogLocation: "stdout", +} + +var Testlog = logger.New(&logConfig) +*/ +func TestAddENIwithPDEnabled(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-1", 1, true, false, false, true) + assert.Error(t, err) + + err = ds.AddENI("eni-2", 2, false, false, false, true) + assert.NoError(t, err) + + assert.Equal(t, len(ds.eniPool), 2) + + eniInfos := ds.GetENIInfos() + assert.Equal(t, len(eniInfos.ENIs), 2) +} + +func TestDeleteENIwithPDEnabled(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-2", 2, false, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-3", 3, false, false, false, true) + assert.NoError(t, err) + + eniInfos := ds.GetENIInfos() + assert.Equal(t, len(eniInfos.ENIs), 3) + + err = ds.RemoveENIFromDataStore("eni-2", false) + assert.NoError(t, err) + + eniInfos = ds.GetENIInfos() + assert.Equal(t, len(eniInfos.ENIs), 2) + + err = ds.RemoveENIFromDataStore("unknown-eni", false) + assert.Error(t, err) + + eniInfos = ds.GetENIInfos() + assert.Equal(t, len(eniInfos.ENIs), 2) + + // Add an IP and assign a pod. + err = ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + ip, device, err := ds.AssignPodIPv4Address(IPAMKey{"net1", "sandbox1", "eth0"}) + assert.NoError(t, err) + assert.Equal(t, "1.1.1.0", ip) + assert.Equal(t, 1, device) + + // Test force removal. The first call fails because eni-1 has an IP with a pod assigned to it, + // but the second call force-removes it and succeeds. + err = ds.RemoveENIFromDataStore("eni-1", false) + assert.Error(t, err) + err = ds.RemoveENIFromDataStore("eni-1", true) + assert.NoError(t, err) +} + +func TestAddENIIPv4Prefix(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-2", 2, false, false, false, true) + assert.NoError(t, err) + + err = ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + var strPrivateIPv4 string + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-1"].IPv4Prefixes["1.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-1", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) + + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-1"].IPv4Prefixes["1.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-1", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + + err = ds.AddIPv4PrefixToStore("eni-2", "2.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Prefixes), 1) + + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-2"].IPv4Prefixes["2.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-2", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + + err = ds.AddIPv4PrefixToStore("dummy-eni", "3.1.1.0/28") + assert.Error(t, err) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + +} + +func TestGetENIIPswithPDEnabled(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-2", 2, false, false, false, true) + assert.NoError(t, err) + + err = ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + var strPrivateIPv4 string + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-1"].IPv4Prefixes["1.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-1", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) + + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-1"].IPv4Prefixes["1.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-1", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + + err = ds.AddIPv4PrefixToStore("eni-2", "2.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Prefixes), 1) + + strPrivateIPv4, err = getFreeIPv4AddrfromPrefix(ds.eniPool["eni-2"].IPv4Prefixes["2.1.1.0"]) + assert.NoError(t, err) + + err = ds.AddPrefixIPv4AddressToStore("eni-2", strPrivateIPv4) + assert.NoError(t, err) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + + eniIPPool, err := ds.GetENIIPs("eni-1") + assert.NoError(t, err) + assert.Equal(t, len(eniIPPool), 2) + + _, err = ds.GetENIIPs("dummy-eni") + assert.Error(t, err) +} + +func TestDelENIIPv4Prefix(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + // Assign a pod. + key := IPAMKey{"net0", "sandbox-1", "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key) + assert.NoError(t, err) + assert.Equal(t, "1.1.1.0", ip) + assert.Equal(t, 1, device) + + // Test force removal. The first call fails because the Prefix has a pod assigned to it, but the + // second call force-removes it and succeeds. + err = ds.DelIPv4PrefixFromStore("eni-1", "1.1.1.0/28", false) + assert.Error(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + err = ds.DelIPv4PrefixFromStore("eni-1", "1.1.1.0/28", true) + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 0) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 0) +} + +func TestPodIPv4AddresswithPDEnabled(t *testing.T) { + checkpoint := NewTestCheckpoint(struct{}{}) + ds := NewDataStore(Testlog, checkpoint) + + err := ds.AddENI("eni-1", 1, true, false, false, true) + assert.NoError(t, err) + + err = ds.AddENI("eni-2", 2, false, false, false, true) + assert.NoError(t, err) + + err = ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + key1 := IPAMKey{"net0", "sandbox-1", "eth0"} + ip, _, err := ds.AssignPodIPv4Address(key1) + + assert.NoError(t, err) + assert.Equal(t, "1.1.1.0", ip) + assert.Equal(t, 1, ds.allocatedPrefix) + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv4Addresses)) + assert.Equal(t, 1, ds.eniPool["eni-1"].AssignedIPv4Addresses()) + assert.Equal(t, checkpoint.Data, &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.0"}, + }, + }) + + podsInfos := ds.AllocatedIPs() + assert.Equal(t, len(podsInfos), 1) + + // duplicate add + ip, _, err = ds.AssignPodIPv4Address(key1) // same id + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.0") + assert.Equal(t, ds.assigned, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 1) + + // Checkpoint error + checkpoint.Error = errors.New("fake checkpoint error") + key2 := IPAMKey{"net0", "sandbox-2", "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2) + assert.Error(t, err) + assert.Equal(t, checkpoint.Data, &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.0"}, + }, + }) + checkpoint.Error = nil + + //1.1.1.1 will be added to cooldown, so we will get another 14 IPS + ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.2") + assert.Equal(t, ds.assigned, 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 2) + + podsInfos = ds.AllocatedIPs() + assert.Equal(t, len(podsInfos), 2) + + key3 := IPAMKey{"net0", "sandbox-3", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key3) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.3") + assert.Equal(t, ds.assigned, 3) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 3) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 3) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 3) + + key4 := IPAMKey{"net0", "sandbox-4", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key4) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.4") + assert.Equal(t, ds.assigned, 4) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 4) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 4) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 4) + + key5 := IPAMKey{"net0", "sandbox-5", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key5) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.5") + assert.Equal(t, ds.assigned, 5) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 5) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 5) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 5) + + key6 := IPAMKey{"net0", "sandbox-6", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key6) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.6") + assert.Equal(t, ds.assigned, 6) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 6) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 6) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 6) + + key7 := IPAMKey{"net0", "sandbox-7", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key7) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.7") + assert.Equal(t, ds.assigned, 7) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 7) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 7) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 7) + + key8 := IPAMKey{"net0", "sandbox-8", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key8) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.8") + assert.Equal(t, ds.assigned, 8) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 8) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 8) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 8) + + key9 := IPAMKey{"net0", "sandbox-9", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key9) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.9") + assert.Equal(t, ds.assigned, 9) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 9) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 9) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 9) + + key10 := IPAMKey{"net0", "sandbox-10", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key10) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.10") + assert.Equal(t, ds.assigned, 10) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 10) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 10) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 10) + + key11 := IPAMKey{"net0", "sandbox-11", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key11) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.11") + assert.Equal(t, ds.assigned, 11) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 11) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 11) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 11) + + key12 := IPAMKey{"net0", "sandbox-12", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key12) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.12") + assert.Equal(t, ds.assigned, 12) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 12) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 12) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 12) + + key13 := IPAMKey{"net0", "sandbox-13", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key13) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.13") + assert.Equal(t, ds.assigned, 13) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 13) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 13) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 13) + + key14 := IPAMKey{"net0", "sandbox-14", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key14) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.14") + assert.Equal(t, ds.assigned, 14) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 14) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 14) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 14) + + key15 := IPAMKey{"net0", "sandbox-15", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key15) + assert.NoError(t, err) + assert.Equal(t, ip, "1.1.1.15") + assert.Equal(t, ds.assigned, 15) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 15) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 15) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 15) + + // no more IP addresses + key16 := IPAMKey{"net0", "sandbox-16", "eth0"} + _, _, err = ds.AssignPodIPv4Address(key16) + assert.Error(t, err) + // Unassign unknown Pod + _, _, _, _, err = ds.UnassignPodIPv4Address(key16) + assert.Error(t, err) + + _, _, deviceNum, _, err := ds.UnassignPodIPv4Address(key2) + assert.NoError(t, err) + assert.Equal(t, ds.assigned, 14) + assert.Equal(t, deviceNum, pod1Ns2Device) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 14) + + //Index will be in cooldown so IP won't be available + _, _, err = ds.AssignPodIPv4Address(key2) + assert.Error(t, err) + + noWarmIPTarget := 0 + noMinimumIPTarget := 0 + noWarmPrefixTarget := 0 + + // Should not be able to free this ENI + eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) + assert.True(t, eni == "") + + ds.eniPool["eni-2"].createTime = time.Time{} + eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) + assert.Equal(t, eni, "eni-2") + + assert.Equal(t, ds.assigned, 14) +} + +func TestWarmPrefixInteractions(t *testing.T) { + ds := NewDataStore(Testlog, NullCheckpoint{}) + + _ = ds.AddENI("eni-1", 1, true, false, false, true) + _ = ds.AddENI("eni-2", 2, false, false, false, true) + _ = ds.AddENI("eni-3", 3, false, false, false, true) + + err := ds.AddIPv4PrefixToStore("eni-1", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Prefixes), 1) + + err = ds.AddIPv4PrefixToStore("eni-2", "2.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Prefixes), 1) + + err = ds.AddIPv4PrefixToStore("eni-3", "1.1.1.0/28") + assert.NoError(t, err) + assert.Equal(t, ds.allocatedPrefix, 3) + assert.Equal(t, len(ds.eniPool["eni-3"].IPv4Prefixes), 1) + + ds.eniPool["eni-2"].createTime = time.Time{} + ds.eniPool["eni-3"].createTime = time.Time{} + + //Setting WARM IP TARGET and ENI TARGET since it should be ignored + // We have three ENIs, 3 prefixes + // We should not be able to remove any ENIs if either warmPrefixTarget >= 3 + eni := ds.RemoveUnusedENIFromStore(1, 1, 3) + assert.Equal(t, "", eni) + + // Should be able to free 2 ENI because prefix target is 1 + removedEni := ds.RemoveUnusedENIFromStore(2, 4, 1) + assert.Contains(t, []string{"eni-2", "eni-3"}, removedEni) + + noWarmPrefixTarget := 0 + // Should be able to free 2 ENI + secondRemovedEni := ds.RemoveUnusedENIFromStore(1, 2, noWarmPrefixTarget) + assert.Contains(t, []string{"eni-2", "eni-3"}, secondRemovedEni) + + assert.NotEqual(t, removedEni, secondRemovedEni, "The two removed ENIs should not be the same ENI.") + + assert.Equal(t, 1, ds.GetENIs()) +} diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 68c7454071..a0ab60e746 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -122,6 +122,13 @@ const ( // vpcENIConfigLabel is used by the VPC resource controller to pick the right ENI config. vpcENIConfigLabel = "vpc.amazonaws.com/eniConfig" + + //envEnableIpv4PrefixDelegation is used to allocate /28 prefix instead of secondary IP for an ENI. + envEnableIpv4PrefixDelegation = "ENABLE_PREFIX_DELEGATION" + + //envWarmPrefixTarget is used to keep a /28 prefix in warm pool. + envWarmPrefixTarget = "WARM_PREFIX_TARGET" + noWarmPrefixTarget = 0 ) var log = logger.Get() @@ -193,20 +200,23 @@ type IPAMContext struct { networkClient networkutils.NetworkAPIs maxIPsPerENI int maxENI int + maxPrefixesPerENI int unmanagedENI int warmENITarget int warmIPTarget int minimumIPTarget int + warmPrefixTarget int primaryIP map[string]string // primaryIP is a map from ENI ID to primary IP of that ENI lastNodeIPPoolAction time.Time lastDecreaseIPPool time.Time // reconcileCooldownCache keeps timestamps of the last time an IP address was unassigned from an ENI, // so that we don't reconcile and add it back too quickly if IMDS lags behind reality. - reconcileCooldownCache ReconcileCooldownCache - terminating int32 // Flag to warn that the pod is about to shut down. - disableENIProvisioning bool - enablePodENI bool - myNodeName string + reconcileCooldownCache ReconcileCooldownCache + terminating int32 // Flag to warn that the pod is about to shut down. + disableENIProvisioning bool + enablePodENI bool + myNodeName string + enableIpv4PrefixDelegation bool } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -288,8 +298,9 @@ func New(k8sapiClient kubernetes.Interface, eniConfig *eniconfig.ENIConfigContro c.networkClient = networkutils.New() c.eniConfig = eniConfig c.useCustomNetworking = UseCustomNetworkCfg() + c.enableIpv4PrefixDelegation = useIpv4PrefixDelegation() - client, err := awsutils.New(c.useCustomNetworking) + client, err := awsutils.New(c.useCustomNetworking, c.enableIpv4PrefixDelegation) if err != nil { return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } @@ -300,9 +311,20 @@ func New(k8sapiClient kubernetes.Interface, eniConfig *eniconfig.ENIConfigContro c.warmENITarget = getWarmENITarget() c.warmIPTarget = getWarmIPTarget() c.minimumIPTarget = getMinimumIPTarget() + c.warmPrefixTarget = getWarmPrefixTarget() c.disableENIProvisioning = disablingENIProvisioning() c.enablePodENI = enablePodENI() + + hypervisorType, err := c.awsClient.GetInstanceHypervisorFamily() + if err != nil { + log.Error("Failed to get hypervisor type") + return nil, err + } + if hypervisorType != "nitro" && c.enableIpv4PrefixDelegation { + log.Error("Prefix delegation is not supported non-nitro instance - " + c.awsClient.GetInstanceType()) + return nil, err + } c.myNodeName = os.Getenv("MY_NODE_NAME") checkpointer := datastore.NewJSONFile(dsBackingStorePath()) c.dataStore = datastore.NewDataStore(log, checkpointer) @@ -338,10 +360,11 @@ func (c *IPAMContext) nodeInit() error { return err } c.maxENI = nodeMaxENI - c.maxIPsPerENI, err = c.awsClient.GetENIIPv4Limit() + c.maxIPsPerENI, c.maxPrefixesPerENI, err = c.GetIPv4Limit() if err != nil { return err } + log.Debugf("Max ip per ENI %d and max prefixes per ENI %d", c.maxIPsPerENI, c.maxPrefixesPerENI) vpcCIDRs, err := c.awsClient.GetVPCIPv4CIDRs() if err != nil { @@ -399,6 +422,14 @@ func (c *IPAMContext) nodeInit() error { return err } + //During upgrade or if prefix delgation knob is disabled to enabled then we + //might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on + if c.enableIpv4PrefixDelegation { + c.tryUnassignIPsFromENIs() + } else { + c.tryUnassignPrefixesFromENIs() + } + if err = c.configureIPRulesForPods(vpcCIDRs); err != nil { return err } @@ -441,7 +472,7 @@ func (c *IPAMContext) nodeInit() error { } // For a new node, attach IPs - increasedPool, err := c.tryAssignIPs() + increasedPool, err := c.tryAssignIPsOrPrefixes() if err == nil && increasedPool { c.updateLastNodeIPPoolAction() } else if err != nil { @@ -507,38 +538,36 @@ func (c *IPAMContext) StartNodeIPPoolManager() { func (c *IPAMContext) updateIPPoolIfRequired() { c.askForTrunkENIIfNeeded() - if c.nodeIPPoolTooLow() { - c.increaseIPPool() - } else if c.nodeIPPoolTooHigh() { - c.decreaseIPPool(decreaseIPPoolInterval) + if c.isDatastorePoolTooLow() { + c.increaseDatastorePool() + } else if c.isDatastorePoolTooHigh() { + c.decreaseDatastorePool(decreaseIPPoolInterval) } - if c.shouldRemoveExtraENIs() { c.tryFreeENI() } } -// decreaseIPPool runs every `interval` and attempts to return unused ENIs and IPs -func (c *IPAMContext) decreaseIPPool(interval time.Duration) { - ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Add(float64(1)) - defer ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Sub(float64(1)) +// decreaseDatastorePool runs every `interval` and attempts to return unused ENIs and IPs +func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) { + ipamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Add(float64(1)) + defer ipamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Sub(float64(1)) now := time.Now() timeSinceLast := now.Sub(c.lastDecreaseIPPool) if timeSinceLast <= interval { - log.Debugf("Skipping decrease IP pool because time since last %v <= %v", timeSinceLast, interval) + log.Debugf("Skipping decrease Datastore pool because time since last %v <= %v", timeSinceLast, interval) return } - log.Debugf("Starting to decrease IP pool") - - c.tryUnassignIPsFromAll() + log.Debugf("Starting to decrease Datastore pool") + c.tryUnassignIPsOrPrefixesFromAll() c.lastDecreaseIPPool = now c.lastNodeIPPoolAction = now - total, used := c.dataStore.GetStats() + total, used, _ := c.dataStore.GetStats() log.Debugf("Successfully decreased IP pool") - logPoolStats(total, used, c.maxIPsPerENI) + logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) } // tryFreeENI always tries to free one ENI @@ -548,7 +577,7 @@ func (c *IPAMContext) tryFreeENI() { return } - eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget) + eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget, c.warmPrefixTarget) if eni == "" { return } @@ -562,13 +591,14 @@ func (c *IPAMContext) tryFreeENI() { } } -// tryUnassignIPsFromAll determines if there are IPs to free when we have extra IPs beyond the target and warmIPTargetDefined +// tryUnassignIPsorPrefixesFromAll determines if there are IPs to free when we have extra IPs beyond the target and warmIPTargetDefined // is enabled, deallocate extra IP addresses -func (c *IPAMContext) tryUnassignIPsFromAll() { - if _, over, warmIPTargetDefined := c.ipTargetState(); warmIPTargetDefined && over > 0 { +func (c *IPAMContext) tryUnassignIPsOrPrefixesFromAll() { + if _, over, warmTargetDefined := c.datastoreTargetState(); warmTargetDefined && over > 0 { eniInfos := c.dataStore.GetENIInfos() for eniID := range eniInfos.ENIs { - ips, err := c.findFreeableIPs(eniID) + //Either returns prefixes or IPs + ips, err := c.findFreeableIPsOrPrefixes(eniID) if err != nil { log.Errorf("Error finding unassigned IPs: %s", err) return @@ -579,56 +609,61 @@ func (c *IPAMContext) tryUnassignIPsFromAll() { } // Delete IPs from datastore - var deletedIPs []string + var deletedIPsOrPrefixes []string for _, toDelete := range ips { // Don't force the delete, since a freeable IP might have been assigned to a pod // before we get around to deleting it. - err := c.dataStore.DelIPv4AddressFromStore(eniID, toDelete, false /* force */) + err := c.dataStore.CleanupDataStore(eniID, toDelete, false /* force */, c.enableIpv4PrefixDelegation) + if err != nil { log.Warnf("Failed to delete IP %s on ENI %s from datastore: %s", toDelete, eniID, err) ipamdErrInc("decreaseIPPool") continue } else { - deletedIPs = append(deletedIPs, toDelete) + deletedIPsOrPrefixes = append(deletedIPsOrPrefixes, toDelete) } } // Deallocate IPs from the instance if they aren't used by pods. - if err := c.awsClient.DeallocIPAddresses(eniID, deletedIPs); err != nil { - log.Warnf("Failed to decrease IP pool by removing IPs %v from ENI %s: %s", deletedIPs, eniID, err) + if err = c.awsClient.DeallocIPAddresses(eniID, deletedIPsOrPrefixes, false); err != nil { + log.Warnf("Failed to decrease pool by removing IPs %v from ENI %s: %s", deletedIPsOrPrefixes, eniID, err) } else { - log.Debugf("Successfully decreased IP pool by removing IPs %v from ENI %s", deletedIPs, eniID) + log.Debugf("Successfully decreased pool by removing IPs %v from ENI %s", deletedIPsOrPrefixes, eniID) } // Track the last time we unassigned IPs from an ENI. We won't reconcile any IPs in this cache // for at least ipReconcileCooldown - c.reconcileCooldownCache.Add(deletedIPs) + c.reconcileCooldownCache.Add(deletedIPsOrPrefixes) } } } -// findFreeableIPs finds and returns IPs that are not assigned to Pods but are attached +// findFreeableIPsOrPrefixes finds and returns IPs that are not assigned to Pods but are attached // to ENIs on the node. -func (c *IPAMContext) findFreeableIPs(eni string) ([]string, error) { - freeableIPs := c.dataStore.FreeableIPs(eni) - +func (c *IPAMContext) findFreeableIPsOrPrefixes(eni string) ([]string, error) { + var freeableIPs []string + if !c.enableIpv4PrefixDelegation { + freeableIPs = c.dataStore.FreeableIPs(eni) + } else if c.enableIpv4PrefixDelegation { + freeableIPs = c.dataStore.FreeablePrefixes(eni) + } // Free the number of IPs `over` the warm IP target, unless `over` is greater than the number of available IPs on // this ENI. In that case we should only free the number of available IPs. - _, over, _ := c.ipTargetState() + _, over, _ := c.datastoreTargetState() numFreeable := min(over, len(freeableIPs)) freeableIPs = freeableIPs[:numFreeable] return freeableIPs, nil } -func (c *IPAMContext) increaseIPPool() { - log.Debug("Starting to increase IP pool size") - ipamdActionsInprogress.WithLabelValues("increaseIPPool").Add(float64(1)) - defer ipamdActionsInprogress.WithLabelValues("increaseIPPool").Sub(float64(1)) +func (c *IPAMContext) increaseDatastorePool() { + log.Debug("Starting to increase pool size") + ipamdActionsInprogress.WithLabelValues("increaseDatastorePool").Add(float64(1)) + defer ipamdActionsInprogress.WithLabelValues("increaseDatastorePool").Sub(float64(1)) - short, _, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined && short == 0 { - log.Debugf("Skipping increase IP pool, warm IP target reached") + short, _, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined && short == 0 { + log.Debugf("Skipping increase Datastore pool, warm target reached") return } @@ -636,9 +671,8 @@ func (c *IPAMContext) increaseIPPool() { log.Debug("AWS CNI is terminating, will not try to attach any new IPs or ENIs right now") return } - // Try to add more IPs to existing ENIs first. - increasedPool, err := c.tryAssignIPs() + increasedPool, err := c.tryAssignIPsOrPrefixes() if err != nil { log.Errorf(err.Error()) } @@ -664,9 +698,13 @@ func (c *IPAMContext) increaseIPPool() { func (c *IPAMContext) updateLastNodeIPPoolAction() { c.lastNodeIPPoolAction = time.Now() - total, used := c.dataStore.GetStats() - log.Debugf("Successfully increased IP pool, total: %d, used: %d", total, used) - logPoolStats(total, used, c.maxIPsPerENI) + total, used, totalPrefix := c.dataStore.GetStats() + if !c.enableIpv4PrefixDelegation { + log.Debugf("Successfully increased IP pool, total: %d, used: %d", total, used) + } else if c.enableIpv4PrefixDelegation { + log.Debugf("Successfully increased Prefix pool, total: %d, used: %d", totalPrefix, used) + } + logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) } func (c *IPAMContext) tryAllocateENI() error { @@ -696,20 +734,25 @@ func (c *IPAMContext) tryAllocateENI() error { return err } - ipsToAllocate := c.maxIPsPerENI - short, _, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined { - ipsToAllocate = short + resourcesToAllocate := c.GetENIResourcesToAllocate() + short, _, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined { + if !c.enableIpv4PrefixDelegation { + resourcesToAllocate = short + } else if c.enableIpv4PrefixDelegation { + resourcesToAllocate = 1 + log.Infof("Update this once we have multiPD support") + } } - err = c.awsClient.AllocIPAddresses(eni, ipsToAllocate) + err = c.awsClient.AllocIPAddresses(eni, resourcesToAllocate) if err != nil { - log.Warnf("Failed to allocate %d IP addresses on an ENI: %v", ipsToAllocate, err) + log.Warnf("Failed to allocate %d IP addresses on an ENI: %v", resourcesToAllocate, err) // Continue to process the allocated IP addresses ipamdErrInc("increaseIPPoolAllocIPAddressesFailed") } - eniMetadata, err := c.awsClient.WaitForENIAndIPsAttached(eni, ipsToAllocate) + eniMetadata, err := c.awsClient.WaitForENIAndIPsAttached(eni, resourcesToAllocate) if err != nil { ipamdErrInc("increaseIPPoolwaitENIAttachedFailed") log.Errorf("Failed to increase pool size: Unable to discover attached ENI from metadata service %v", err) @@ -726,15 +769,22 @@ func (c *IPAMContext) tryAllocateENI() error { return err } -// For an ENI, try to fill in missing IPs on an existing ENI -func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { - // If WARM_IP_TARGET is set, only proceed if we are short of target - short, _, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined && short == 0 { +// For an ENI, try to fill in missing IPs on an existing ENI with PD disabled +// try to fill in missing Prefixes on an existing ENI with PD enabled +func (c *IPAMContext) tryAssignIPsOrPrefixes() (increasedPool bool, err error) { + short, _, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined && short == 0 { + log.Infof("Warm target set and short is 0 so not assigning IPs or Prefixes") return false, nil } + if !c.enableIpv4PrefixDelegation { + return c.tryAssignIPs() + } else { + return c.tryAssignPrefixes() + } +} - // Find an ENI where we can add more IPs +func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { eni := c.dataStore.GetENINeedsIP(c.maxIPsPerENI, c.useCustomNetworking) if eni != nil && len(eni.IPv4Addresses) < c.maxIPsPerENI { currentNumberOfAllocatedIPs := len(eni.IPv4Addresses) @@ -755,12 +805,39 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { ipamdErrInc("increaseIPPoolGetENIaddressesFailed") return true, errors.Wrap(err, "failed to get ENI IP addresses during IP allocation") } + c.addENIaddressesToDataStore(ec2Addrs, eni.ID) return true, nil } return false, nil } +func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) { + eni := c.dataStore.GetENINeedsIP(c.maxPrefixesPerENI, c.useCustomNetworking) + if eni != nil { + currentNumberOfAllocatedPrefixes := len(eni.IPv4Prefixes) + + err = c.awsClient.AllocIPAddresses(eni.ID, c.maxPrefixesPerENI-currentNumberOfAllocatedPrefixes) + if err != nil { + log.Warnf("failed to allocate all available IPv4 Prefixes on ENI %s, err: %v", eni.ID, err) + // Try to just get one more prefix + err = c.awsClient.AllocIPAddresses(eni.ID, 1) + if err != nil { + ipamdErrInc("increaseIPPoolAllocIPAddressesFailed") + return false, errors.Wrap(err, fmt.Sprintf("failed to allocate one IPv4 prefix on ENI %s, err: %v", eni.ID, err)) + } + } + ec2Prefixes, err := c.awsClient.GetIPv4PrefixesFromEC2(eni.ID) + if err != nil { + ipamdErrInc("increaseIPPoolGetENIprefixedFailed") + return true, errors.Wrap(err, "failed to get ENI Prefix addresses during IP allocation") + } + c.addENIprefixesToDataStore(ec2Prefixes, eni.ID) + return true, nil + } + return false, nil +} + // setupENI does following: // 1) add ENI to datastore // 2) set up linux ENI related networking stack. @@ -768,7 +845,7 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isTrunkENI, isEFAENI bool) error { primaryENI := c.awsClient.GetPrimaryENI() // Add the ENI to the datastore - err := c.dataStore.AddENI(eni, eniMetadata.DeviceNumber, eni == primaryENI, isTrunkENI, isEFAENI) + err := c.dataStore.AddENI(eni, eniMetadata.DeviceNumber, eni == primaryENI, isTrunkENI, isEFAENI, c.enableIpv4PrefixDelegation) if err != nil && err.Error() != datastore.DuplicatedENIError { return errors.Wrapf(err, "failed to add ENI %s to data store", eni) } @@ -789,7 +866,19 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT } } + if c.enableIpv4PrefixDelegation && len(eniMetadata.IPv4Addresses) > 1 { + //During upgrade or when PD knob is disabled->enabled then SIPs will be attached hence add to DS + log.Infof("Found ENIs having secondary IPs while PD is enabled") + } else if !c.enableIpv4PrefixDelegation && len(eniMetadata.IPv4Prefixes) > 0 { + //When PD mode is toggled from enabled->disabled + log.Infof("Found ENIs having prefixes while PD is disabled") + } + //Either case add the IPs and prefixes to datastore. Reconiler will ensure + //if pd enabled then reconcile only prefix + //if pd not enabled then reconcile only SIPs. c.addENIaddressesToDataStore(eniMetadata.IPv4Addresses, eni) + c.addENIprefixesToDataStore(eniMetadata.IPv4Prefixes, eni) + return nil } @@ -806,10 +895,24 @@ func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfac ipamdErrInc("addENIaddressesToDataStoreAddENIIPv4AddressFailed") } } - total, assigned := c.dataStore.GetStats() + total, assigned, _ := c.dataStore.GetStats() log.Debugf("IP Address Pool stats: total: %d, assigned: %d", total, assigned) } +func (c *IPAMContext) addENIprefixesToDataStore(ec2Addrs []*ec2.Ipv4PrefixSpecification, eni string) { + log.Debugf("In addENIPrefixedToDataStore for %s and no of prefixes %d", eni, len(ec2Addrs)) + for _, ec2Addr := range ec2Addrs { + err := c.dataStore.AddIPv4PrefixToStore(eni, aws.StringValue(ec2Addr.Ipv4Prefix)) + if err != nil && err.Error() != datastore.IPAlreadyInStoreError { + log.Warnf("Failed to increase IP pool, failed to add IP %s to data store", ec2Addr) + // continue to add next address + ipamdErrInc("addENIaddressesToDataStoreAddENIIPv4PrefixFailed") + } + } + total, assigned, totalPrefix := c.dataStore.GetStats() + log.Debugf("Prefix Address Pool stats: total: %d, assigned: %d, total prefixes: %d", total, assigned, totalPrefix) +} + // getMaxENI returns the maximum number of ENIs to attach to this instance. This is calculated as the lesser of // the limit for the instance type and the value configured via the MAX_ENI environment variable. If the value of // the environment variable is 0 or less, it will be ignored and the maximum for the instance is returned. @@ -850,8 +953,29 @@ func getWarmENITarget() int { return defaultWarmENITarget } -func logPoolStats(total, used, maxAddrsPerENI int) { - log.Debugf("IP pool stats: total = %d, used = %d, c.maxIPsPerENI = %d", total, used, maxAddrsPerENI) +func getWarmPrefixTarget() int { + inputStr, found := os.LookupEnv(envWarmPrefixTarget) + + if !found { + return noWarmPrefixTarget + } + + if input, err := strconv.Atoi(inputStr); err == nil { + if input < 0 { + return noWarmPrefixTarget + } + log.Debugf("Using WARM_PREFIX_TARGET %v", input) + return input + } + return noWarmPrefixTarget +} + +func logPoolStats(total int, used int, maxAddrsPerENI int, Ipv4PrefixDelegation bool) { + if !Ipv4PrefixDelegation { + log.Debugf("IP pool stats: total = %d, used = %d, c.maxIPsPerENI = %d", total, used, maxAddrsPerENI) + } else { + log.Debugf("Prefix pool stats: total = %d, used = %d, c.maxIPsPerENI = %d", total, used, maxAddrsPerENI) + } } func (c *IPAMContext) askForTrunkENIIfNeeded() { @@ -870,49 +994,28 @@ func (c *IPAMContext) askForTrunkENIIfNeeded() { } } -// nodeIPPoolTooLow returns true if IP pool is below low threshold -func (c *IPAMContext) nodeIPPoolTooLow() bool { - short, _, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined { - return short > 0 - } - - total, used := c.dataStore.GetStats() - - available := total - used - poolTooLow := available < c.maxIPsPerENI*c.warmENITarget || (c.warmENITarget == 0 && available == 0) - if poolTooLow { - logPoolStats(total, used, c.maxIPsPerENI) - log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) - } - return poolTooLow -} - -// nodeIPPoolTooHigh returns true if IP pool is above high threshold -func (c *IPAMContext) nodeIPPoolTooHigh() bool { - _, over, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined { - return over > 0 - } - - // We only ever report the pool being too high if WARM_IP_TARGET is set - return false -} - // shouldRemoveExtraENIs returns true if we should attempt to find an ENI to free. When WARM_IP_TARGET is set, we // always check and do verification in getDeletableENI() func (c *IPAMContext) shouldRemoveExtraENIs() bool { - _, _, warmIPTargetDefined := c.ipTargetState() - if warmIPTargetDefined { + _, _, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined { return true } - total, used := c.dataStore.GetStats() + total, used, totalPrefix := c.dataStore.GetStats() available := total - used - // We need the +1 to make sure we are not going below the WARM_ENI_TARGET. - shouldRemoveExtra := available >= (c.warmENITarget+1)*c.maxIPsPerENI + var shouldRemoveExtra bool + if !c.enableIpv4PrefixDelegation { + // We need the +1 to make sure we are not going below the WARM_ENI_TARGET. + shouldRemoveExtra = available >= (c.warmENITarget+1)*c.maxIPsPerENI + } else if c.enableIpv4PrefixDelegation { + // All Ips are available so see if we need to remove some ENIs + _, maxIpsPerPrefix, _ := datastore.GetPrefixDelegationDefaults() + shouldRemoveExtra = available == (totalPrefix * maxIpsPerPrefix) + log.Debugf("Total available IPs %d and prefixes available %d", available, totalPrefix) + } if shouldRemoveExtra { - logPoolStats(total, used, c.maxIPsPerENI) + logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) log.Debugf("It might be possible to remove extra ENIs because available (%d) >= (ENI target (%d) + 1) * addrsPerENI (%d): ", available, c.warmENITarget, c.maxIPsPerENI) } return shouldRemoveExtra @@ -992,15 +1095,28 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { // Mark phase for _, attachedENI := range attachedENIs { - eniIPPool, err := c.dataStore.GetENIIPs(attachedENI.ENIID) - if err == nil { - // If the attached ENI is in the data store - log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID) - // Reconcile IP pool - c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID) - // Mark action, remove this ENI from currentENIs map - delete(currentENIs, attachedENI.ENIID) - continue + if !c.enableIpv4PrefixDelegation { + eniIPPool, err := c.dataStore.GetENIIPs(attachedENI.ENIID) + if err == nil { + // If the attached ENI is in the data store + log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID) + // Reconcile IP pool + c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID) + // Mark action, remove this ENI from currentENIs map + delete(currentENIs, attachedENI.ENIID) + continue + } + } else { + eniPrefixPool, err := c.dataStore.GetENIPrefixes(attachedENI.ENIID) + if err == nil { + // If the attached ENI is in the data store + log.Debugf("Reconcile existing ENI %s IP prefixes", attachedENI.ENIID) + // Reconcile IP pool + c.eniPrefixPoolReconcile(eniPrefixPool, attachedENI, attachedENI.ENIID) + // Mark action, remove this ENI from currentENIs map + delete(currentENIs, attachedENI.ENIID) + continue + } } // Add new ENI @@ -1030,8 +1146,8 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { reconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc() } log.Debug("Successfully Reconciled ENI/IP pool") - total, assigned := c.dataStore.GetStats() - log.Debugf("IP Address Pool stats: total: %d, assigned: %d", total, assigned) + total, assigned, totalPrefix := c.dataStore.GetStats() + log.Debugf("IP/Prefix Address Pool stats: total: %d, assigned: %d, total prefixes: %d", total, assigned, totalPrefix) c.lastNodeIPPoolAction = curTime } @@ -1075,6 +1191,47 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E } } +func (c *IPAMContext) eniPrefixPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string) { + attachedENIIPs := attachedENI.IPv4Prefixes + needEC2Reconcile := true + // Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API. + log.Debugf("Found prefix pool count %d for eni %s\n", len(ipPool), eni) + + if len(ipPool) != len(attachedENIIPs) { + log.Warnf("Instance metadata does not match data store! ipPool: %v, metadata: %v", ipPool, attachedENIIPs) + log.Debugf("We need to check the ENI status by calling the EC2 control plane.") + // Call EC2 to verify IPs on this ENI + ec2Addresses, err := c.awsClient.GetIPv4PrefixesFromEC2(eni) + if err != nil { + log.Errorf("Failed to fetch ENI IP addresses! Aborting reconcile of ENI %s", eni) + return + } + attachedENIIPs = ec2Addresses + needEC2Reconcile = false + } + + // Add all known attached IPs to the datastore + seenIPs := c.verifyAndAddPrefixesToDatastore(eni, attachedENIIPs, needEC2Reconcile) + + // Sweep phase, delete remaining Prefixes since they should not remain in the datastore + for _, existingIP := range ipPool { + if seenIPs[existingIP] { + continue + } + + log.Debugf("Reconcile and delete Prefix %s on ENI %s", existingIP, eni) + // Force the delete, since we have verified with EC2 that these secondary IPs are no longer assigned to this ENI + err := c.dataStore.DelIPv4PrefixFromStore(eni, existingIP, true /* force */) + if err != nil { + log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err) + ipamdErrInc("ipReconcileDel") + // continue instead of bailout due to one ip + continue + } + reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileDel"}).Inc() + } +} + // verifyAndAddIPsToDatastore updates the datastore with the known secondary IPs. IPs who are out of cooldown gets added // back to the datastore after being verified against EC2. func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []*ec2.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool) map[string]bool { @@ -1145,6 +1302,75 @@ func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []*e return seenIPs } +// verifyAndAddPrefixesToDatastore updates the datastore with the known Prefixes. Prefixes who are out of cooldown gets added +// back to the datastore after being verified against EC2. +func (c *IPAMContext) verifyAndAddPrefixesToDatastore(eni string, attachedENIIPs []*ec2.Ipv4PrefixSpecification, needEC2Reconcile bool) map[string]bool { + var ec2VerifiedAddresses []*ec2.Ipv4PrefixSpecification + seenIPs := make(map[string]bool) + for _, privateIPv4 := range attachedENIIPs { + strPrivateIPv4 := aws.StringValue(privateIPv4.Ipv4Prefix) + log.Debugf("Check in coolddown Found prefix %s", strPrivateIPv4) + + // Check if this IP was recently freed + found, recentlyFreed := c.reconcileCooldownCache.RecentlyFreed(strPrivateIPv4) + if found { + log.Debugf("found in cooldown") + if recentlyFreed { + log.Debugf("Reconcile skipping IP %s on ENI %s because it was recently unassigned from the ENI.", strPrivateIPv4, eni) + continue + } else { + if needEC2Reconcile { + // IMDS data might be stale + log.Debugf("This IP was recently freed, but is now out of cooldown. We need to verify with EC2 control plane.") + // Only call EC2 once for this ENI + if ec2VerifiedAddresses == nil { + var err error + // Call EC2 to verify IPs on this ENI + ec2VerifiedAddresses, err = c.awsClient.GetIPv4PrefixesFromEC2(eni) + if err != nil { + log.Errorf("Failed to fetch ENI IP addresses from EC2! %v", err) + // Do not delete this IP from the datastore or cooldown until we have confirmed with EC2 + seenIPs[strPrivateIPv4] = true + continue + } + } + // Verify that the IP really belongs to this ENI + isReallyAttachedToENI := false + for _, ec2Addr := range ec2VerifiedAddresses { + if strPrivateIPv4 == aws.StringValue(ec2Addr.Ipv4Prefix) { + isReallyAttachedToENI = true + log.Debugf("Verified that IP %s is attached to ENI %s", strPrivateIPv4, eni) + break + } + } + if !isReallyAttachedToENI { + log.Warnf("Skipping IP %s on ENI %s because it does not belong to this ENI!", strPrivateIPv4, eni) + continue + } + } + // The IP can be removed from the cooldown cache + // TODO: Here we could check if the IP is still used by a pod stuck in Terminating state. (Issue #1091) + c.reconcileCooldownCache.Remove(strPrivateIPv4) + } + } + + // Try to add the IP + err := c.dataStore.AddIPv4PrefixToStore(eni, strPrivateIPv4) + if err != nil && err.Error() != datastore.IPAlreadyInStoreError { + log.Errorf("Failed to reconcile IP %s on ENI %s", strPrivateIPv4, eni) + ipamdErrInc("ipReconcileAdd") + // Continue to check the other IPs instead of bailout due to one wrong IP + continue + + } + // Mark action + seenIPs[strPrivateIPv4] = true + log.Infof("Marked %s as seen", strPrivateIPv4) + reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileAdd"}).Inc() + } + return seenIPs +} + // UseCustomNetworkCfg returns whether Pods needs to use pod specific configuration or not. func UseCustomNetworkCfg() bool { if strValue := os.Getenv(envCustomNetworkCfg); strValue != "" { @@ -1204,6 +1430,10 @@ func enablePodENI() bool { return getEnvBoolWithDefault(envEnablePodENI, false) } +func useIpv4PrefixDelegation() bool { + return getEnvBoolWithDefault(envEnableIpv4PrefixDelegation, false) +} + // filterUnmanagedENIs filters out ENIs marked with the "node.k8s.amazonaws.com/no_manage" tag func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutils.ENIMetadata { numFiltered := 0 @@ -1227,30 +1457,48 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil return ret } -// ipTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, -// accounting for the MINIMUM_IP_TARGET -func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) { - if c.warmIPTarget == noWarmIPTarget && c.minimumIPTarget == noMinimumIPTarget { +// datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, +// accounting for the MINIMUM_IP_TARGET without prefix delegation enabled. +// With prefix delegation this function accounts for WARM_PREFIX_TARGET +func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) { + if c.warmIPTarget == noWarmIPTarget && c.minimumIPTarget == noMinimumIPTarget && !c.enableIpv4PrefixDelegation { // there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI return 0, 0, false } - total, assigned := c.dataStore.GetStats() + if c.warmPrefixTarget == noWarmPrefixTarget && c.enableIpv4PrefixDelegation { + return 0, 0, false + } + + total, assigned, totalPrefix := c.dataStore.GetStats() available := total - assigned - // short is greater than 0 when we have fewer available IPs than the warm IP target - short = max(c.warmIPTarget-available, 0) + if !c.enableIpv4PrefixDelegation { + // short is greater than 0 when we have fewer available IPs than the warm IP target + short = max(c.warmIPTarget-available, 0) - // short is greater than the warm IP target alone when we have fewer total IPs than the minimum target - short = max(short, c.minimumIPTarget-total) + // short is greater than the warm IP target alone when we have fewer total IPs than the minimum target + short = max(short, c.minimumIPTarget-total) - // over is the number of available IPs we have beyond the warm IP target - over = max(available-c.warmIPTarget, 0) + // over is the number of available IPs we have beyond the warm IP target + over = max(available-c.warmIPTarget, 0) - // over is less than the warm IP target alone if it would imply reducing total IPs below the minimum target - over = max(min(over, total-c.minimumIPTarget), 0) + // over is less than the warm IP target alone if it would imply reducing total IPs below the minimum target + over = max(min(over, total-c.minimumIPTarget), 0) - log.Debugf("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", c.warmIPTarget, total, assigned, available, short, over) + log.Debugf("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", c.warmIPTarget, total, assigned, available, short, over) + } else { + //With prefix delegation short/over will be in terms of prefixes + //TODO - See if this can be optimized. If there are holes in the subnet, can it be considered as 1 prefix, + //for instance prefix 1 -> used [8] free [8], prefix 2 -> used [8] free [8], available = 16, so if warm prefix target is 2, then we shouldnt allocate + //one more prefix. + freePrefixes := c.dataStore.GetFreePrefixes() + + short = max(c.warmPrefixTarget-freePrefixes, 0) + over = max(freePrefixes-c.warmPrefixTarget, 0) + + log.Debugf("Current warm prefix IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d, total prefixes: %d", c.warmPrefixTarget, total, assigned, available, short, over, totalPrefix) + } return short, over, true } @@ -1344,3 +1592,135 @@ func (c *IPAMContext) SetNodeLabel(key, value string) error { func (c *IPAMContext) GetPod(podName, namespace string) (*v1.Pod, error) { return c.k8sClient.CoreV1().Pods(namespace).Get(podName, metav1.GetOptions{}) } + +func (c *IPAMContext) tryUnassignIPsFromENIs() { + eniInfos := c.dataStore.GetENIInfos() + for eniID := range eniInfos.ENIs { + freeableIPs := c.dataStore.FreeableIPs(eniID) + + if len(freeableIPs) == 0 { + continue + } + + // Delete IPs from datastore + var deletedIPs []string + for _, toDelete := range freeableIPs { + // Don't force the delete, since a freeable IP might have been assigned to a pod + // before we get around to deleting it. + err := c.dataStore.DelIPv4AddressFromStore(eniID, toDelete, false /* force */) + if err != nil { + log.Warnf("Failed to delete IP %s on ENI %s from datastore: %s", toDelete, eniID, err) + ipamdErrInc("decreaseIPPool") + continue + } else { + deletedIPs = append(deletedIPs, toDelete) + } + } + + // Deallocate IPs from the instance if they aren't used by pods. + if err := c.awsClient.DeallocIPAddresses(eniID, deletedIPs, true); err != nil { + log.Warnf("Failed to decrease IP pool by removing IPs %v from ENI %s: %s", deletedIPs, eniID, err) + } else { + log.Debugf("Successfully decreased IP pool by removing IPs %v from ENI %s", deletedIPs, eniID) + } + } +} + +func (c *IPAMContext) tryUnassignPrefixesFromENIs() { + eniInfos := c.dataStore.GetENIInfos() + for eniID := range eniInfos.ENIs { + c.tryUnassignPrefixFromENI(eniID) + } +} + +func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) { + FreeablePrefixes := c.dataStore.FreeablePrefixes(eniID) + if len(FreeablePrefixes) == 0 { + return + } + // Delete Prefixes from datastore + var deletedPrefixes []string + for _, toDelete := range FreeablePrefixes { + // Don't force the delete, since a freeable Prefix might have been assigned to a pod + // before we get around to deleting it. + err := c.dataStore.DelIPv4PrefixFromStore(eniID, toDelete, false /* force */) + if err != nil { + log.Warnf("Failed to delete Prefix %s on ENI %s from datastore: %s", toDelete, eniID, err) + ipamdErrInc("decreaseIPPool") + return + } else { + deletedPrefixes = append(deletedPrefixes, toDelete) + } + } + + // Deallocate IPs from the instance if they aren't used by pods. + if err := c.awsClient.DeallocPrefixAddresses(eniID, deletedPrefixes); err != nil { + log.Warnf("Failed to delete prefix %v from ENI %s: %s", deletedPrefixes, eniID, err) + } else { + log.Debugf("Successfully prefix removing IPs %v from ENI %s", deletedPrefixes, eniID) + } +} + +func (c *IPAMContext) GetENIResourcesToAllocate() int { + if !c.enableIpv4PrefixDelegation { + return c.maxIPsPerENI + } else { + return c.maxPrefixesPerENI + } +} + +func (c *IPAMContext) GetIPv4Limit() (int, int, error) { + var maxIPsPerENI, maxPrefixesPerENI, maxIpsPerPrefix int + var err error + if !c.enableIpv4PrefixDelegation { + maxIPsPerENI, err = c.awsClient.GetENIIPv4Limit() + maxPrefixesPerENI = 0 + if err != nil { + return 0, 0, err + } + } else if c.enableIpv4PrefixDelegation { + //Single PD - allocate one prefix per ENI and new add will be new ENI + prefix + //Multi - allocate one prefix per ENI and new add will be new prefix or new ENI + prefix + maxPrefixesPerENI, maxIpsPerPrefix, _ = datastore.GetPrefixDelegationDefaults() + maxIPsPerENI = maxPrefixesPerENI * maxIpsPerPrefix + log.Debugf("max prefix %d max ips %d", maxPrefixesPerENI, maxIPsPerENI) + } + return maxIPsPerENI, maxPrefixesPerENI, nil +} + +func (c *IPAMContext) isDatastorePoolTooLow() bool { + short, _, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined { + return short > 0 + } + + total, used, _ := c.dataStore.GetStats() + + available := total - used + if !c.enableIpv4PrefixDelegation { + poolTooLow := available < c.maxIPsPerENI*c.warmENITarget || (c.warmENITarget == 0 && available == 0) + if poolTooLow { + logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, c.warmENITarget, c.maxIPsPerENI) + } + return poolTooLow + } else { + _, maxIpsPerPrefix, _ := datastore.GetPrefixDelegationDefaults() + poolTooLow := available < maxIpsPerPrefix*c.warmPrefixTarget || (c.warmPrefixTarget == 0 && available == 0) + if poolTooLow { + logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + log.Debugf("Prefix pool is too low: available (%d) < Warm IP target (%d) * maxIpsPerPrefix (%d)", available, c.warmPrefixTarget, maxIpsPerPrefix) + } + return poolTooLow + } +} + +func (c *IPAMContext) isDatastorePoolTooHigh() bool { + _, over, warmTargetDefined := c.datastoreTargetState() + if warmTargetDefined { + return over > 0 + } + + // We only ever report the pool being too high if WARM_IP_TARGET or WARM_PREFIX_TARGET is set + return false +} diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index b4b4806797..15fa24666e 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -54,6 +54,10 @@ const ( ipaddr12 = "10.10.20.12" vpcCIDR = "10.10.0.0/16" myNodeName = "testNodeName" + prefix01 = "10.10.30.0/28" + prefix02 = "10.10.40.0/28" + ipaddrPD01 = "10.10.30.0" + ipaddrPD02 = "10.10.40.0" ) type testMocks struct { @@ -155,6 +159,83 @@ func TestNodeInit(t *testing.T) { assert.NoError(t, err) } +func TestNodeInitwithPDenabled(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + fakeCheckpoint := datastore.CheckpointData{ + Version: datastore.CheckpointFormatVersion, + Allocations: []datastore.CheckpointEntry{ + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddrPD01}, + }, + } + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + k8sClient: m.clientset, + maxIPsPerENI: 16, + maxPrefixesPerENI: 1, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), + myNodeName: myNodeName, + enableIpv4PrefixDelegation: true, + } + mockContext.dataStore.CheckpointMigrationPhase = 2 + + eni1, eni2 := getDummyENIMetadataWithPrefix() + + var cidrs []string + m.awsutils.EXPECT().GetENILimit().Return(4, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv4Prefixes, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(eni2.ENIID).AnyTimes().Return(eni2.IPv4Prefixes, nil) + m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().IsUnmanagedENI(eni2.ENIID).Return(false).AnyTimes() + + primaryIP := net.ParseIP(ipaddr01) + m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) + m.awsutils.EXPECT().GetPrimaryENImac().Return("") + m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false).Return(nil) + + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + + eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} + resp := awsutils.DescribeAllENIsResult{ + ENIMetadata: eniMetadataSlice, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + } + m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) + + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP) + + var rules []netlink.Rule + m.network.EXPECT().GetRuleList().Return(rules, nil) + + m.network.EXPECT().UseExternalSNAT().Return(false) + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any(), gomock.Any(), true) + + fakeNode := v1.Node{ + TypeMeta: metav1.TypeMeta{Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{}, + } + _, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) + + // Add Prefixes + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) + + err := mockContext.nodeInit() + assert.NoError(t, err) +} + func getDummyENIMetadata() (awsutils.ENIMetadata, awsutils.ENIMetadata) { primary := true notPrimary := false @@ -194,6 +275,42 @@ func getDummyENIMetadata() (awsutils.ENIMetadata, awsutils.ENIMetadata) { return eni1, eni2 } +func getDummyENIMetadataWithPrefix() (awsutils.ENIMetadata, awsutils.ENIMetadata) { + primary := true + testAddr1 := ipaddr01 + testPrefix1 := prefix01 + testAddr2 := ipaddr11 + eni1 := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &testPrefix1, + }, + }, + } + + eni2 := awsutils.ENIMetadata{ + ENIID: secENIid, + MAC: secMAC, + DeviceNumber: secDevice, + SubnetIPv4CIDR: secSubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr2, Primary: &primary, + }, + }, + } + return eni1, eni2 +} + func TestIncreaseIPPoolDefault(t *testing.T) { _ = os.Unsetenv(envCustomNetworkCfg) testIncreaseIPPool(t, false) @@ -283,7 +400,104 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) m.awsutils.EXPECT().AllocIPAddresses(eni2, 14) - mockContext.increaseIPPool() + mockContext.increaseDatastorePool() +} + +func TestIncreasePrefixPoolDefault(t *testing.T) { + _ = os.Unsetenv(envCustomNetworkCfg) + testIncreasePrefixPool(t, false) +} + +func TestIncreasePrefixPoolCustomENI(t *testing.T) { + _ = os.Setenv(envCustomNetworkCfg, "true") + testIncreasePrefixPool(t, true) +} + +func testIncreasePrefixPool(t *testing.T, useENIConfig bool) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + maxIPsPerENI: 16, + maxPrefixesPerENI: 1, + maxENI: 4, + warmENITarget: 1, + networkClient: m.network, + useCustomNetworking: UseCustomNetworkCfg(), + eniConfig: m.eniconfig, + primaryIP: make(map[string]string), + terminating: int32(0), + enableIpv4PrefixDelegation: true, + } + + mockContext.dataStore = testDatastore() + + primary := true + testAddr1 := ipaddr01 + testAddr11 := ipaddr11 + testPrefix1 := prefix01 + testPrefix2 := prefix02 + eni2 := secENIid + + podENIConfig := &v1alpha1.ENIConfigSpec{ + SecurityGroups: []string{"sg1-id", "sg2-id"}, + Subnet: "subnet1", + } + var sg []*string + + for _, sgID := range podENIConfig.SecurityGroups { + sg = append(sg, aws.String(sgID)) + } + + if useENIConfig { + m.eniconfig.EXPECT().MyENIConfig().Return(podENIConfig, nil) + m.awsutils.EXPECT().AllocENI(true, sg, podENIConfig.Subnet).Return(eni2, nil) + } else { + m.awsutils.EXPECT().AllocENI(false, nil, "").Return(eni2, nil) + } + + eniMetadata := []awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &testPrefix1, + }, + }, + }, + { + ENIID: secENIid, + MAC: secMAC, + DeviceNumber: secDevice, + SubnetIPv4CIDR: secSubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr11, Primary: &primary, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &testPrefix2, + }, + }, + }, + } + + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 1).Return(eniMetadata[1], nil) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) + m.awsutils.EXPECT().AllocIPAddresses(eni2, 1) + + mockContext.increaseDatastorePool() } func TestTryAddIPToENI(t *testing.T) { @@ -349,7 +563,7 @@ func TestTryAddIPToENI(t *testing.T) { m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - mockContext.increaseIPPool() + mockContext.increaseDatastorePool() } func TestNodeIPPoolReconcile(t *testing.T) { @@ -448,6 +662,98 @@ func TestNodeIPPoolReconcile(t *testing.T) { assert.Equal(t, 0, curENIs.TotalIPs) } +func TestNodePrefixPoolReconcile(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enableIpv4PrefixDelegation: true, + } + + mockContext.dataStore = testDatastore() + + primary := true + primaryENIMetadata := getPrimaryENIMetadataPDenabled() + testAddr1 := *primaryENIMetadata.IPv4Prefixes[0].Ipv4Prefix + // Always the primary ENI + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).AnyTimes().Return(false) + eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} + m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) + resp := awsutils.DescribeAllENIsResult{ + ENIMetadata: eniMetadataList, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + } + m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + + mockContext.nodeIPPoolReconcile(0) + + curENIs := mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) + + // 1 prefix lost in IMDS + oneIPUnassigned := []awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + IPv4Prefixes: make([]*ec2.Ipv4PrefixSpecification, 0), + }, + } + m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Prefixes, nil) + + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 0, curENIs.TotalIPs) + + // New ENI attached + newENIMetadata := getSecondaryENIMetadataPDenabled() + + twoENIs := append(oneIPUnassigned, newENIMetadata) + + // Two ENIs found + m.awsutils.EXPECT().GetAttachedENIs().Return(twoENIs, nil) + m.awsutils.EXPECT().IsUnmanagedENI(secENIid).Times(2).Return(false) + resp2 := awsutils.DescribeAllENIsResult{ + ENIMetadata: twoENIs, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + } + m.awsutils.EXPECT().DescribeAllENIs().Return(resp2, nil) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet) + + mockContext.nodeIPPoolReconcile(0) + + // Verify that we now have 2 ENIs, primary ENI with 0 prefixes, and secondary ENI with 1 prefix + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 2, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) + + // Remove the secondary ENI in the IMDS metadata + m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) + + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 0, curENIs.TotalIPs) +} + func TestGetWarmENITarget(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -465,6 +771,23 @@ func TestGetWarmENITarget(t *testing.T) { assert.Equal(t, warmIPTarget, noWarmIPTarget) } +func TestGetWarmPrefixTarget(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + _ = os.Setenv("WARM_PREFIX_TARGET", "5") + warmPrefixTarget := getWarmPrefixTarget() + assert.Equal(t, warmPrefixTarget, 5) + + _ = os.Unsetenv("WARM_PREFIX_TARGET") + warmPrefixTarget = getWarmPrefixTarget() + assert.Equal(t, warmPrefixTarget, noWarmPrefixTarget) + + _ = os.Setenv("WARM_PREFIX_TARGET", "non-integer-string") + warmPrefixTarget = getWarmPrefixTarget() + assert.Equal(t, warmPrefixTarget, noWarmPrefixTarget) +} + func TestGetWarmIPTargetState(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -478,21 +801,21 @@ func TestGetWarmIPTargetState(t *testing.T) { mockContext.dataStore = testDatastore() - _, _, warmIPTargetDefined := mockContext.ipTargetState() + _, _, warmIPTargetDefined := mockContext.datastoreTargetState() assert.False(t, warmIPTargetDefined) mockContext.warmIPTarget = 5 - short, over, warmIPTargetDefined := mockContext.ipTargetState() + short, over, warmIPTargetDefined := mockContext.datastoreTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, 5, short) assert.Equal(t, 0, over) // add 2 addresses to datastore - _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false) + _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false, false) _ = mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.1") _ = mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.2") - short, over, warmIPTargetDefined = mockContext.ipTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, 3, short) assert.Equal(t, 0, over) @@ -502,12 +825,60 @@ func TestGetWarmIPTargetState(t *testing.T) { _ = mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.4") _ = mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.5") - short, over, warmIPTargetDefined = mockContext.ipTargetState() + short, over, warmIPTargetDefined = mockContext.datastoreTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, 0, short) assert.Equal(t, 0, over) } +func TestGetWarmPrefixTargetState(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enableIpv4PrefixDelegation: true, + } + + mockContext.dataStore = testDatastore() + + _, _, warmPrefixTargetDefined := mockContext.datastoreTargetState() + assert.False(t, warmPrefixTargetDefined) + + mockContext.warmPrefixTarget = 5 + short, over, warmPrefixTargetDefined := mockContext.datastoreTargetState() + assert.True(t, warmPrefixTargetDefined) + assert.Equal(t, 5, short) + assert.Equal(t, 0, over) + + // add 2 addresses to datastore + _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false, true) + _ = mockContext.dataStore.AddIPv4PrefixToStore("eni-1", "10.1.1.0/28") + _ = mockContext.dataStore.AddENI("eni-2", 2, true, false, false, true) + _ = mockContext.dataStore.AddIPv4PrefixToStore("eni-1", "20.1.1.0/28") + + short, over, warmPrefixTargetDefined = mockContext.datastoreTargetState() + assert.True(t, warmPrefixTargetDefined) + assert.Equal(t, 3, short) + assert.Equal(t, 0, over) + + //Add 3 more + _ = mockContext.dataStore.AddENI("eni-3", 3, true, false, false, true) + _ = mockContext.dataStore.AddIPv4PrefixToStore("eni-3", "30.1.1.0/28") + _ = mockContext.dataStore.AddENI("eni-4", 4, true, false, false, true) + _ = mockContext.dataStore.AddIPv4PrefixToStore("eni-4", "40.1.1.0/28") + _ = mockContext.dataStore.AddENI("eni-5", 5, true, false, false, true) + _ = mockContext.dataStore.AddIPv4PrefixToStore("eni-5", "50.1.1.0/28") + + short, over, warmPrefixTargetDefined = mockContext.datastoreTargetState() + assert.True(t, warmPrefixTargetDefined) + assert.Equal(t, 0, short) + assert.Equal(t, 0, over) +} + func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -536,17 +907,61 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &IPAMContext{ - awsClient: m.awsutils, - dataStore: tt.fields.datastore, - useCustomNetworking: false, - eniConfig: m.eniconfig, - networkClient: m.network, - maxIPsPerENI: tt.fields.maxIPsPerENI, - maxENI: -1, - warmENITarget: tt.fields.warmENITarget, - warmIPTarget: tt.fields.warmIPTarget, + awsClient: m.awsutils, + dataStore: tt.fields.datastore, + useCustomNetworking: false, + eniConfig: m.eniconfig, + networkClient: m.network, + maxIPsPerENI: tt.fields.maxIPsPerENI, + maxENI: -1, + warmENITarget: tt.fields.warmENITarget, + warmIPTarget: tt.fields.warmIPTarget, + enableIpv4PrefixDelegation: false, + } + if got := c.isDatastorePoolTooLow(); got != tt.want { + t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + type fields struct { + maxIPsPerENI int + warmPrefixTarget int + datastore *datastore.DataStore + } + + tests := []struct { + name string + fields fields + want bool + }{ + {"Test new ds, all defaults", fields{16, 1, testDatastore()}, true}, + {"Test new ds, 0 ENIs", fields{16, 0, testDatastore()}, true}, + {"Test 3 unused IPs, 1 warm", fields{16, 1, datastoreWithFreeIPsFromPrefix()}, false}, + {"Test 1 used, 1 warm Prefix", fields{16, 1, datastoreWith1Pod1FromPrefix()}, true}, + {"Test 1 used, 0 warm Prefix", fields{16, 0, datastoreWith1Pod1FromPrefix()}, false}, + {"Test 3 used, 1 warm Prefix", fields{16, 1, datastoreWith3PodsFromPrefix()}, true}, + {"Test 3 used, 0 warm Prefix", fields{16, 0, datastoreWith3PodsFromPrefix()}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &IPAMContext{ + awsClient: m.awsutils, + dataStore: tt.fields.datastore, + useCustomNetworking: false, + eniConfig: m.eniconfig, + networkClient: m.network, + maxIPsPerENI: tt.fields.maxIPsPerENI, + maxENI: -1, + warmPrefixTarget: tt.fields.warmPrefixTarget, + enableIpv4PrefixDelegation: true, } - if got := c.nodeIPPoolTooLow(); got != tt.want { + if got := c.isDatastorePoolTooLow(); got != tt.want { t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) } }) @@ -561,7 +976,7 @@ func testDatastore() *datastore.DataStore { func datastoreWith3FreeIPs() *datastore.DataStore { datastoreWith3FreeIPs := testDatastore() - _ = datastoreWith3FreeIPs.AddENI(primaryENIid, 1, true, false, false) + _ = datastoreWith3FreeIPs.AddENI(primaryENIid, 1, true, false, false, false) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr01) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr02) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr03) @@ -593,6 +1008,38 @@ func datastoreWith3Pods() *datastore.DataStore { return datastoreWith3Pods } +func datastoreWithFreeIPsFromPrefix() *datastore.DataStore { + datastoreWithFreeIPs := testDatastore() + _ = datastoreWithFreeIPs.AddENI(primaryENIid, 1, true, false, false, true) + _ = datastoreWithFreeIPs.AddIPv4PrefixToStore(primaryENIid, prefix01) + return datastoreWithFreeIPs +} + +func datastoreWith1Pod1FromPrefix() *datastore.DataStore { + datastoreWith1Pod1 := datastoreWithFreeIPsFromPrefix() + + _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{ + NetworkName: "net0", + ContainerID: "sandbox-1", + IfName: "eth0", + }) + return datastoreWith1Pod1 +} + +func datastoreWith3PodsFromPrefix() *datastore.DataStore { + datastoreWith3Pods := datastoreWithFreeIPsFromPrefix() + + for i := 0; i < 3; i++ { + key := datastore.IPAMKey{ + NetworkName: "net0", + ContainerID: fmt.Sprintf("sandbox-%d", i), + IfName: "eth0", + } + _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key) + } + return datastoreWith3Pods +} + func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { ctrl := gomock.NewController(t) @@ -689,7 +1136,7 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress // Add ENI and IPs to datastore eniID := primaryENIMetadata.ENIID - _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false) + _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false, false) mockContext.primaryIP[eniID] = testAddr1 mockContext.addENIaddressesToDataStore(primaryENIMetadata.IPv4Addresses, eniID) curENIs := mockContext.dataStore.GetENIInfos() @@ -756,6 +1203,89 @@ func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { assert.Equal(t, 2, curENIs.TotalIPs) } +func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enableIpv4PrefixDelegation: true, + } + + mockContext.dataStore = testDatastore() + + primaryENIMetadata := getPrimaryENIMetadataPDenabled() + testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress + // Add ENI and IPs to datastore + eniID := primaryENIMetadata.ENIID + _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false, true) + mockContext.primaryIP[eniID] = testAddr1 + mockContext.addENIprefixesToDataStore(primaryENIMetadata.IPv4Prefixes, eniID) + curENIs := mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) + eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} + m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) + m.awsutils.EXPECT().IsUnmanagedENI(eniID).Return(false).AnyTimes() + + // First reconcile, IMDS returns correct IPs so no change needed + mockContext.nodeIPPoolReconcile(0) + + // IMDS returns no prefixes, the EC2 call fails + primary := true + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + }, + }, nil) + + // eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed")) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) + + // IMDS returns no prefixes + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + }, + }, nil) + + // eniIPPoolReconcile() calls EC2 to get the actual count that should still be 16 + m.awsutils.EXPECT().GetIPv4PrefixesFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Prefixes, nil) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) + + // If no ENI is found, we abort the reconcile + m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 16, curENIs.TotalIPs) +} func getPrimaryENIMetadata() awsutils.ENIMetadata { primary := true notPrimary := false @@ -805,6 +1335,54 @@ func getSecondaryENIMetadata() awsutils.ENIMetadata { return newENIMetadata } +func getPrimaryENIMetadataPDenabled() awsutils.ENIMetadata { + primary := true + testAddr1 := ipaddr01 + testPrefix1 := prefix01 + + eniMetadata := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &testPrefix1, + }, + }, + } + return eniMetadata +} + +func getSecondaryENIMetadataPDenabled() awsutils.ENIMetadata { + primary := true + testAddr3 := ipaddr11 + testPrefix2 := prefix02 + + newENIMetadata := awsutils.ENIMetadata{ + ENIID: secENIid, + MAC: secMAC, + DeviceNumber: secDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr3, Primary: &primary, + }, + }, + IPv4Prefixes: []*ec2.Ipv4PrefixSpecification{ + { + Ipv4Prefix: &testPrefix2, + }, + }, + } + return newENIMetadata +} + func TestIPAMContext_setupENI(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -851,6 +1429,52 @@ func TestIPAMContext_setupENI(t *testing.T) { assert.Equal(t, 1, len(mockContext.primaryIP)) } +func TestIPAMContext_setupENIwithPDenabled(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + } + //mockContext.primaryIP[] + + mockContext.dataStore = testDatastore() + primary := true + notPrimary := false + testAddr1 := ipaddr01 + testAddr2 := ipaddr02 + primaryENIMetadata := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + { + PrivateIpAddress: &testAddr2, Primary: ¬Primary, + }, + }, + } + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + err := mockContext.setupENI(primaryENIMetadata.ENIID, primaryENIMetadata, false, false) + assert.NoError(t, err) + // Primary ENI added + assert.Equal(t, 1, len(mockContext.primaryIP)) + + newENIMetadata := getSecondaryENIMetadata() + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2")) + + err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, false, false) + assert.Error(t, err) + assert.Equal(t, 1, len(mockContext.primaryIP)) +} + func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { m := setup(t) defer m.ctrl.Finish() @@ -875,7 +1499,7 @@ func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { } _, _ = m.clientset.CoreV1().Nodes().Create(&fakeNode) - _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false) + _ = mockContext.dataStore.AddENI("eni-1", 1, true, false, false, false) // If ENABLE_POD_ENI is not set, nothing happens mockContext.askForTrunkENIIfNeeded() diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index cee3bb6344..8e049f0408 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -195,7 +195,24 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp IfName: in.IfName, NetworkName: in.NetworkName, } - ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPv4Address(ipamKey) + eni, ip, deviceNumber, mismatchedStore, err := s.ipamContext.dataStore.UnassignPodIPv4Address(ipamKey) + if eni != nil && mismatchedStore { + if s.ipamContext.enableIpv4PrefixDelegation { + addr := eni.IPv4Addresses[ip] + if addr != nil && addr.Prefix == nil { + log.Debugf("IP belongs to secondary pool with PD enabled so free from EC2") + var deletedIPs []string + deletedIPs = append(deletedIPs, ip) + if err = s.ipamContext.awsClient.DeallocIPAddresses(eni.ID, deletedIPs, !s.ipamContext.enableIpv4PrefixDelegation); err != nil { + log.Warnf("Failed to remove IP %v from ENI %s: %s", deletedIPs, eni.ID, err) + } else { + log.Debugf("Successfully removed IPs %v from ENI %s", deletedIPs, eni.ID) + } + } + } else { + s.ipamContext.tryUnassignPrefixFromENI(eni.ID) + } + } if err == datastore.ErrUnknownPod && s.ipamContext.enablePodENI { pod, err := s.ipamContext.GetPod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE) diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 42957c68a5..a474acf5e7 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -10,6 +10,8 @@ ENV GOPROXY=direct # Copy modules in before the rest of the source to only expire cache on module changes: COPY go.mod go.sum ./ +COPY vendor/ vendor/ + RUN go mod download COPY Makefile ./ diff --git a/scripts/dockerfiles/Dockerfile.test b/scripts/dockerfiles/Dockerfile.test index 2f518d60a1..0c50717436 100644 --- a/scripts/dockerfiles/Dockerfile.test +++ b/scripts/dockerfiles/Dockerfile.test @@ -17,6 +17,7 @@ RUN go get -u golang.org/x/tools/cmd/goimports # go.mod and go.sum go into their own layers. COPY go.mod . COPY go.sum . +COPY vendor/ vendor/ # This ensures `go mod download` happens only when go.mod and go.sum change. RUN go mod download diff --git a/scripts/ec2_model_override/cleanup.sh b/scripts/ec2_model_override/cleanup.sh new file mode 100755 index 0000000000..967eaf8cf2 --- /dev/null +++ b/scripts/ec2_model_override/cleanup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf ./vendor +go mod edit -dropreplace github.com/aws/aws-sdk-go +go mod tidy diff --git a/scripts/ec2_model_override/setup.sh b/scripts/ec2_model_override/setup.sh new file mode 100755 index 0000000000..4b733b86f1 --- /dev/null +++ b/scripts/ec2_model_override/setup.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +mkdir -p ./vendor/github.com/aws + +SDK_MODEL_SOURCE=/home/varavaj/AWS/tmp/release-automation/staging-sdk211076960/sdk/src/github.com/aws/aws-sdk-go/models/apis/ec2/2016-11-15 +SDK_VENDOR_PATH=./vendor/github.com/aws/aws-sdk-go +API_VERSION=2016-11-15 +API_PATH=$SDK_VENDOR_PATH/models/apis/ec2/$API_VERSION +SERVICE_NAME=$([ "$EC2_PREVIEW" == "y" ] && echo "pd-preview" || echo "pdmesh" ) + +# Clone the SDK to the vendor path (removing an old one if necessary) +rm -rf $SDK_VENDOR_PATH +git clone --depth 1 https://github.com/aws/aws-sdk-go.git $SDK_VENDOR_PATH + +# Override the SDK models for App Mesh +#curl -s $SDK_MODEL_SOURCE/api.json |\ + # Always use the "vanilla" flavors of UID, Service ID, and Service Name. + # This ensures the SDK we generate is always the same interface objects, only changing + # the endpoint and signing name when using preview. +# jq "(.metadata.uid) |= \"appmesh-$API_VERSION\"" |\ +# jq "(.metadata.serviceId) |= \"App Mesh\"" |\ +# jq "(.metadata.serviceFullName) |= \"AWS App Mesh\"" |\ + # Set the endpoint prefix and signing name to the desired value, based on + # whether or not we're using the preview endpoint +# jq "(.metadata | .endpointPrefix, .signingName) |= \"$SERVICE_NAME\"" \ +# > $API_PATH/api-2.json +#curl -s $SDK_MODEL_SOURCE/docs.json > $API_PATH/docs-2.json +#curl -s $SDK_MODEL_SOURCE/examples.json > $API_PATH/examples-1.json +#curl -s $SDK_MODEL_SOURCE/paginators.json > $API_PATH/paginators-1.json + +cp $SDK_MODEL_SOURCE/api-2.json $API_PATH/api-2.json +cp $SDK_MODEL_SOURCE/docs-2.json $API_PATH/docs-2.json +cp $SDK_MODEL_SOURCE/examples-1.json $API_PATH/examples-1.json +cp $SDK_MODEL_SOURCE/paginators-1.json $API_PATH/paginators-1.json +# Generate the SDK + +pushd ./vendor/github.com/aws/aws-sdk-go +make generate +popd + +# Use the vendored version of aws-sdk-go +go mod edit -replace github.com/aws/aws-sdk-go=./vendor/github.com/aws/aws-sdk-go +go mod tidy diff --git a/scripts/gen_vpc_ip_limits.go b/scripts/gen_vpc_ip_limits.go index 6656f621af..9ef783f2d8 100644 --- a/scripts/gen_vpc_ip_limits.go +++ b/scripts/gen_vpc_ip_limits.go @@ -75,8 +75,9 @@ func main() { instanceType := aws.StringValue(info.InstanceType) eniLimit := int(aws.Int64Value(info.NetworkInfo.MaximumNetworkInterfaces)) ipv4Limit := int(aws.Int64Value(info.NetworkInfo.Ipv4AddressesPerInterface)) + hypervisorType := aws.StringValue(info.Hypervisor) if instanceType != "" && eniLimit > 0 && ipv4Limit > 0 { - eniLimitMap[instanceType] = awsutils.InstanceTypeLimits{ENILimit: eniLimit, IPv4Limit: ipv4Limit} + eniLimitMap[instanceType] = awsutils.InstanceTypeLimits{ENILimit: eniLimit, IPv4Limit: ipv4Limit, HypervisorType: hypervisorType} } } // Paginate to the next request @@ -144,16 +145,16 @@ func main() { // addManualLimits has the list of faulty or missing instance types func addManualLimits(limitMap map[string]awsutils.InstanceTypeLimits) map[string]awsutils.InstanceTypeLimits { manuallyAddedLimits := map[string]awsutils.InstanceTypeLimits{ - "cr1.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "hs1.8xlarge": {ENILimit: 8, IPv4Limit: 30}, - "u-12tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "u-18tb1.metal": {ENILimit: 15, IPv4Limit: 50}, - "u-24tb1.metal": {ENILimit: 15, IPv4Limit: 50}, - "u-6tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "u-9tb1.metal": {ENILimit: 5, IPv4Limit: 30}, - "c5a.metal": {ENILimit: 15, IPv4Limit: 50}, - "c5ad.metal": {ENILimit: 15, IPv4Limit: 50}, - "p4d.24xlarge": {ENILimit: 15, IPv4Limit: 50}, + "cr1.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "unkown"}, + "hs1.8xlarge": {ENILimit: 8, IPv4Limit: 30, HypervisorType: "unkown"}, + "u-12tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "u-18tb1.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "u-24tb1.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "u-6tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "u-9tb1.metal": {ENILimit: 5, IPv4Limit: 30, HypervisorType: "unkown"}, + "c5a.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "c5ad.metal": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, + "p4d.24xlarge": {ENILimit: 15, IPv4Limit: 50, HypervisorType: "unkown"}, } for instanceType, instanceLimits := range manuallyAddedLimits { val, ok := limitMap[instanceType]