Skip to content

Commit

Permalink
Add support for specifying PrivateIpv4Address via annotation (#1762)
Browse files Browse the repository at this point in the history 
* Add support for specifying PrivateIpv4Address via annotation

Signed-off-by: Alex Williams <[email protected]>

* Find IP my matching CIDR. Change annoation.

* Rename ipForSubnet and change if structure

* Catch ip parsing errors. Add test. Fmt changes
  • Loading branch information
@Smirl
Smirl authored Feb 19, 2021
1 parent 478f171 commit dd8f8c6
Show file tree
Hide file tree
Showing 6 changed files with 325 additions and 43 deletions.
53 changes: 27 additions & 26 deletions docs/guide/service/annotations.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,33 @@
- json: `"{ \"key\": \"value\" }"`

## Annotations
| Name | Type | Default | Notes |
|--------------------------------------------------------------------------------|------------|---------------------------|------------------------|
| service.beta.kubernetes.io/aws-load-balancer-type | string | | |
| service.beta.kubernetes.io/aws-load-balancer-internal | boolean | false | |
| [service.beta.kubernetes.io/aws-load-balancer-proxy-protocol](#proxy-protocol-v2) | string | | Set to `"*"` to enable |
| service.beta.kubernetes.io/aws-load-balancer-ip-address-type | string | ipv4 | ipv4 \| dualstack |
| service.beta.kubernetes.io/aws-load-balancer-access-log-enabled | boolean | false | |
| service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name | string | | |
| service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix | string | | |
| service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled | boolean | false | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-cert | stringList | | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-ports | stringList | | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy | string | ELBSecurityPolicy-2016-08 | |
| service.beta.kubernetes.io/aws-load-balancer-backend-protocol | string | | |
| service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags | stringMap | | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold | integer | 3 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold | integer | 3 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout | integer | 10 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval | integer | 10 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol | string | TCP | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-port | integer \| traffic-port | traffic-port | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-path | string | "/" for HTTP(S) protocols | |
| service.beta.kubernetes.io/aws-load-balancer-eip-allocations | stringList | | |
| [service.beta.kubernetes.io/aws-load-balancer-target-group-attributes](#target-group-attributes) | stringMap | | |
| [service.beta.kubernetes.io/aws-load-balancer-subnets](#subnets) | stringList | | |
| [service.beta.kubernetes.io/aws-load-balancer-alpn-policy](#alpn-policy) | stringList | | |
| Name | Type | Default | Notes |
|--------------------------------------------------------------------------------------------------|-------------------------|---------------------------|--------------------------------------------------------|
| service.beta.kubernetes.io/aws-load-balancer-type | string | | |
| service.beta.kubernetes.io/aws-load-balancer-internal | boolean | false | |
| [service.beta.kubernetes.io/aws-load-balancer-proxy-protocol](#proxy-protocol-v2) | string | | Set to `"*"` to enable |
| service.beta.kubernetes.io/aws-load-balancer-ip-address-type | string | ipv4 | ipv4 \| dualstack |
| service.beta.kubernetes.io/aws-load-balancer-access-log-enabled | boolean | false | |
| service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-name | string | | |
| service.beta.kubernetes.io/aws-load-balancer-access-log-s3-bucket-prefix | string | | |
| service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled | boolean | false | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-cert | stringList | | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-ports | stringList | | |
| service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy | string | ELBSecurityPolicy-2016-08 | |
| service.beta.kubernetes.io/aws-load-balancer-backend-protocol | string | | |
| service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags | stringMap | | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold | integer | 3 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold | integer | 3 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout | integer | 10 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval | integer | 10 | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol | string | TCP | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-port | integer \| traffic-port | traffic-port | |
| service.beta.kubernetes.io/aws-load-balancer-healthcheck-path | string | "/" for HTTP(S) protocols | |
| service.beta.kubernetes.io/aws-load-balancer-eip-allocations | stringList | | Public Facing lb only. Length/order must match subnets |
| service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses | stringList | | Internal lb only. Length/order must match subnets |
| [service.beta.kubernetes.io/aws-load-balancer-target-group-attributes](#target-group-attributes) | stringMap | | |
| [service.beta.kubernetes.io/aws-load-balancer-subnets](#subnets) | stringList | | |
| [service.beta.kubernetes.io/aws-load-balancer-alpn-policy](#alpn-policy) | stringList | | |


## Traffic Routing
Expand Down
1 change: 1 addition & 0 deletions pkg/annotations/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ const (
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
Expand Down
2 changes: 1 addition & 1 deletion pkg/networking/subnet_resolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,4 +341,4 @@ func (r *defaultSubnetsResolver) checkSubnetIsNotTaggedForOtherClusters(subnet *
return false
}
return true
}
}
7 changes: 3 additions & 4 deletions pkg/networking/subnet_resolver_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ func Test_defaultSubnetsResolver_ResolveViaDiscovery(t *testing.T) {
{
name: "subnet with cluster tag gets precedence",
fields: fields{
vpcID: "vpc-1",
vpcID: "vpc-1",
clusterName: "kube-cluster",
describeSubnetsAsListCalls: []describeSubnetsAsListCall{
{
Expand Down Expand Up @@ -453,7 +453,7 @@ func Test_defaultSubnetsResolver_ResolveViaDiscovery(t *testing.T) {
{
name: "subnets tagged for some other clusters get ignored",
fields: fields{
vpcID: "vpc-1",
vpcID: "vpc-1",
clusterName: "kube-cluster",
describeSubnetsAsListCalls: []describeSubnetsAsListCall{
{
Expand Down Expand Up @@ -551,7 +551,7 @@ func Test_defaultSubnetsResolver_ResolveViaDiscovery(t *testing.T) {
{
name: "subnets with multiple cluster tags",
fields: fields{
vpcID: "vpc-1",
vpcID: "vpc-1",
clusterName: "kube-cluster",
describeSubnetsAsListCalls: []describeSubnetsAsListCall{
{
Expand Down Expand Up @@ -610,7 +610,6 @@ func Test_defaultSubnetsResolver_ResolveViaDiscovery(t *testing.T) {
},
},
},

},
}

Expand Down
61 changes: 54 additions & 7 deletions pkg/service/model_build_load_balancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,16 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net"
"regexp"
"strconv"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/pkg/errors"
"regexp"
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
"strconv"
)

const (
Expand Down Expand Up @@ -46,7 +48,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem
if err != nil {
return elbv2model.LoadBalancerSpec{}, err
}
subnetMappings, err := t.buildLoadBalancerSubnetMappings(ctx, t.ec2Subnets)
subnetMappings, err := t.buildLoadBalancerSubnetMappings(ctx, scheme, t.ec2Subnets)
if err != nil {
return elbv2model.LoadBalancerSpec{}, err
}
Expand Down Expand Up @@ -109,25 +111,70 @@ func (t *defaultModelBuildTask) buildLoadBalancerTags(ctx context.Context) (map[
return t.buildAdditionalResourceTags(ctx)
}

func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(_ context.Context, ec2Subnets []*ec2.Subnet) ([]elbv2model.SubnetMapping, error) {
func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Context, scheme elbv2model.LoadBalancerScheme, ec2Subnets []*ec2.Subnet) ([]elbv2model.SubnetMapping, error) {
var eipAllocation []string
eipConfigured := t.annotationParser.ParseStringSliceAnnotation(annotations.SvcLBSuffixEIPAllocations, &eipAllocation, t.service.Annotations)
if eipConfigured && len(eipAllocation) != len(ec2Subnets) {
return []elbv2model.SubnetMapping{}, errors.Errorf("number of EIP allocations (%d) and subnets (%d) must match", len(eipAllocation), len(ec2Subnets))
var privateIpv4Addresses []string
ipv4Configured := t.annotationParser.ParseStringSliceAnnotation(annotations.SvcLBSuffixPrivateIpv4Addresses, &privateIpv4Addresses, t.service.Annotations)

// Validation
if eipConfigured && ipv4Configured {
return []elbv2model.SubnetMapping{}, errors.Errorf("only one of EIP allocations or PrivateIpv4Addresses can be set")
}
if eipConfigured {
if scheme == elbv2model.LoadBalancerSchemeInternal {
return []elbv2model.SubnetMapping{}, errors.Errorf("EIP allocations can only be set for internet facing load balancers")
} else if len(eipAllocation) != len(ec2Subnets) {
return []elbv2model.SubnetMapping{}, errors.Errorf("number of EIP allocations (%d) and subnets (%d) must match", len(eipAllocation), len(ec2Subnets))
}
}
if ipv4Configured {
if scheme == elbv2model.LoadBalancerSchemeInternetFacing {
return []elbv2model.SubnetMapping{}, errors.Errorf("PrivateIpv4Addresses can only be set for internal balancers")
} else if len(privateIpv4Addresses) != len(ec2Subnets) {
return []elbv2model.SubnetMapping{}, errors.Errorf("number of PrivateIpv4Addresses (%d) and subnets (%d) must match", len(privateIpv4Addresses), len(ec2Subnets))
}
}

subnetMappings := make([]elbv2model.SubnetMapping, 0, len(ec2Subnets))
for idx, subnet := range ec2Subnets {
mapping := elbv2model.SubnetMapping{
SubnetID: aws.StringValue(subnet.SubnetId),
}
if idx < len(eipAllocation) {
if eipConfigured {
mapping.AllocationID = aws.String(eipAllocation[idx])
}
if ipv4Configured {
ip, err := t.getMatchingIPforSubnet(ctx, subnet, privateIpv4Addresses)
if err != nil {
return []elbv2model.SubnetMapping{}, err
}
mapping.PrivateIPv4Address = aws.String(ip)
}
subnetMappings = append(subnetMappings, mapping)
}
return subnetMappings, nil
}

// Return the ip address which is in the subnet. Error if not match
// Can be extended for ipv6 if required
func (t *defaultModelBuildTask) getMatchingIPforSubnet(_ context.Context, subnet *ec2.Subnet, privateIpv4Addresses []string) (string, error) {
_, ipv4Net, err := net.ParseCIDR(*subnet.CidrBlock)
if err != nil {
return "", errors.Wrap(err, "subnet CIDR block could not be parsed")
}
for _, ipString := range privateIpv4Addresses {
ip := net.ParseIP(ipString)
if ip == nil {
return "", errors.Errorf("cannot parse ip %s", ipString)
}
if ipv4Net.Contains(ip) {
return ipString, nil
}
}
return "", errors.Errorf("no matching ip for subnet %s", *subnet.SubnetId)
}

func (t *defaultModelBuildTask) resolveLoadBalancerSubnets(ctx context.Context, scheme elbv2model.LoadBalancerScheme) ([]*ec2.Subnet, error) {
var rawSubnetNameOrIDs []string
if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.SvcLBSuffixSubnets, &rawSubnetNameOrIDs, t.service.Annotations); exists {
Expand Down
Loading

0 comments on commit dd8f8c6

Please sign in to comment.