Skip to content

Commit

Permalink
Add support for federated services to the service mirror controller (#…
Browse files Browse the repository at this point in the history
…13269)

When the service mirror controller detects a service in the remote cluster which matches the federated service selector (`mirror.linkerd.io/federated=memeber` by default), it will add that service to the federated service in the local cluster named `<svc>-federated`, creating this service if it does not already exist.  To join a service to a federated service, it is added to the `multicluster.linkerd.io/remote-discovery` annotation on the federated service which contains a comma separated list of values in the form `<svc>@<cluster>`.  When a remote service no longer exists or matches the federated service selector, it is removed from the federated service by removing it from the `mutlicluster.linkerd.io/remote-discovery` annotation.

We also add a new `local-service-mirror` deployment to the Linkerd-multicluster extension which watches the local cluster for any services which match the federated service selector.  Any services in the local cluster which match will be added to the federated service by setting the `mutlicluster.linkerd.io/local-discovery` annotation on the federated service to the local service name.

Signed-off-by: Alex Leong <[email protected]>
  • Loading branch information
adleong authored Nov 8, 2024
1 parent c66f83e commit 50b6a17
Show file tree
Hide file tree
Showing 24 changed files with 1,628 additions and 271 deletions.
17 changes: 9 additions & 8 deletions controller/k8s/test_helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/metadata/fake"
Expand All @@ -14,40 +15,40 @@ import (

// NewFakeAPI provides a mock Kubernetes API for testing.
func NewFakeAPI(configs ...string) (*API, error) {
clientSet, _, _, spClientSet, err := k8s.NewFakeClientSets(configs...)
clientSet, _, _, spClientSet, dynamicClient, err := k8s.NewFakeClientSets(configs...)
if err != nil {
return nil, err
}

return NewFakeClusterScopedAPI(clientSet, spClientSet), nil
return NewFakeClusterScopedAPI(clientSet, spClientSet, dynamicClient), nil
}

// NewFakeAPI provides a mock Kubernetes API for testing.
func NewFakeAPIWithActions(configs ...string) (*API, func() []testing.Action, error) {
clientSet, _, _, spClientSet, err := k8s.NewFakeClientSets(configs...)
clientSet, _, _, spClientSet, dynamicClient, err := k8s.NewFakeClientSets(configs...)
if err != nil {
return nil, nil, err
}

return NewFakeClusterScopedAPI(clientSet, spClientSet), clientSet.Actions, nil
return NewFakeClusterScopedAPI(clientSet, spClientSet, dynamicClient), clientSet.Actions, nil
}

// NewFakeAPIWithL5dClient provides a mock Kubernetes API for testing like
// NewFakeAPI, but it also returns the mock client for linkerd CRDs
func NewFakeAPIWithL5dClient(configs ...string) (*API, l5dcrdclient.Interface, error) {
clientSet, _, _, l5dClientSet, err := k8s.NewFakeClientSets(configs...)
clientSet, _, _, l5dClientSet, dynamicClient, err := k8s.NewFakeClientSets(configs...)
if err != nil {
return nil, nil, err
}

return NewFakeClusterScopedAPI(clientSet, l5dClientSet), l5dClientSet, nil
return NewFakeClusterScopedAPI(clientSet, l5dClientSet, dynamicClient), l5dClientSet, nil
}

// NewFakeClusterScopedAPI provides a mock Kubernetes API for testing.
func NewFakeClusterScopedAPI(clientSet kubernetes.Interface, l5dClientSet l5dcrdclient.Interface) *API {
func NewFakeClusterScopedAPI(clientSet kubernetes.Interface, l5dClientSet l5dcrdclient.Interface, dynamicClient dynamic.Interface) *API {
return NewClusterScopedAPI(
clientSet,
nil,
dynamicClient,
l5dClientSet,
"fake",
CJ,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ rules:
- apiGroups: ["multicluster.linkerd.io"]
resources: ["links"]
verbs: ["list", "get", "watch"]
- apiGroups: ["multicluster.linkerd.io"]
resources: ["links/status"]
verbs: ["update"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["create", "get", "update", "patch"]
Expand Down
13 changes: 13 additions & 0 deletions multicluster/charts/linkerd-multicluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,25 @@ Kubernetes: `>=1.22.0-0`
| imagePullSecrets | list | `[]` | For Private docker registries, authentication is needed. Registry secrets are applied to the respective service accounts |
| linkerdNamespace | string | `"linkerd"` | Namespace of linkerd installation |
| linkerdVersion | string | `"linkerdVersionValue"` | Control plane version |
| localServiceMirror.GID | int | `2103` | Group id under which the Service Mirror shall be ran |
| localServiceMirror.UID | int | `2103` | User id under which the Service Mirror shall be ran |
| localServiceMirror.enablePprof | bool | `false` | enables the use of pprof endpoints on control plane component's admin servers |
| localServiceMirror.federatedServiceSelector | string | `"mirror.linkerd.io/federated=member"` | Label selector for federated service members in the local cluster. |
| localServiceMirror.image.name | string | `"cr.l5d.io/linkerd/controller"` | Docker image for the Service mirror component (uses the Linkerd controller image) |
| localServiceMirror.image.pullPolicy | string | imagePullPolicy | Pull policy for the Service mirror container image |
| localServiceMirror.image.version | string | linkerdVersion | Tag for the Service mirror container image |
| localServiceMirror.logFormat | string | `"plain"` | Log format (`plain` or `json`) |
| localServiceMirror.logLevel | string | `"info"` | Log level for the Multicluster components |
| localServiceMirror.replicas | int | `1` | Number of local service mirror replicas to run |
| localServiceMirror.resources | object | `{}` | Resources for the Service mirror container |
| localServiceMirror.serviceMirrorRetryLimit | int | `3` | Number of times local service mirror updates are allowed to be requeued (retried) |
| namespaceMetadata.image.name | string | `"extension-init"` | Docker image name for the namespace-metadata instance |
| namespaceMetadata.image.pullPolicy | string | imagePullPolicy | Pull policy for the namespace-metadata instance |
| namespaceMetadata.image.registry | string | `"cr.l5d.io/linkerd"` | Docker registry for the namespace-metadata instance |
| namespaceMetadata.image.tag | string | `"v0.1.1"` | Docker image tag for the namespace-metadata instance |
| namespaceMetadata.nodeSelector | object | `{}` | Node selectors for the namespace-metadata instance |
| namespaceMetadata.tolerations | list | `[]` | Tolerations for the namespace-metadata instance |
| podAnnotations | object | `{}` | Additional annotations to add to all pods |
| podLabels | object | `{}` | Additional labels to add to all pods |
| proxyOutboundPort | int | `4140` | The port on which the proxy accepts outbound traffic |
| remoteMirrorServiceAccount | bool | `true` | If the remote mirror service account should be installed |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: linkerd-local-service-mirror-access-local-resources
labels:
linkerd.io/extension: multicluster
component: local-service-mirror
{{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
rules:
- apiGroups: [""]
resources: ["endpoints", "services"]
verbs: ["list", "get", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "get", "watch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["create", "get", "update", "patch"]
- apiGroups: ["multicluster.linkerd.io"]
resources: ["links"]
verbs: ["list", "get", "watch"]
- apiGroups: ["multicluster.linkerd.io"]
resources: ["links/status"]
verbs: ["update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: linkerd-local-service-mirror-access-local-resources
labels:
linkerd.io/extension: multicluster
component: local-service-mirror
{{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: linkerd-local-service-mirror-access-local-resources
subjects:
- kind: ServiceAccount
name: linkerd-local-service-mirror
namespace: {{.Release.Namespace}}
---
kind: ServiceAccount
apiVersion: v1
metadata:
name: linkerd-local-service-mirror
namespace: {{ .Release.Namespace }}
labels:
linkerd.io/extension: multicluster
component: local-service-mirror
{{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
{{- include "partials.image-pull-secrets" .Values.imagePullSecrets }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
linkerd.io/extension: multicluster
component: local-service-mirror
{{- with .Values.commonLabels }}{{ toYaml . | trim | nindent 4 }}{{- end }}
name: linkerd-local-service-mirror
namespace: {{ .Release.Namespace }}
spec:
replicas: {{ .Values.localServiceMirror.replicas }}
revisionHistoryLimit: {{.Values.revisionHistoryLimit}}
selector:
matchLabels:
component: local-service-mirror
{{- if .Values.enablePodAntiAffinity }}
strategy:
rollingUpdate:
maxUnavailable: 1
{{- end }}
template:
metadata:
annotations:
linkerd.io/inject: enabled
cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
config.alpha.linkerd.io/proxy-wait-before-exit-seconds: "0"
{{- with .Values.podAnnotations }}{{ toYaml . | trim | nindent 8 }}{{- end }}
labels:
linkerd.io/extension: multicluster
component: local-service-mirror
{{- with .Values.podLabels }}{{ toYaml . | trim | nindent 8 }}{{- end }}
spec:
{{- if .Values.enablePodAntiAffinity}}
{{- with $tree := deepCopy . }}
{{- $_ := set $tree "component" "local-service-mirror" -}}
{{- $_ := set $tree "label" "component" -}}
{{- include "linkerd.affinity" $tree | nindent 6 }}
{{- end }}
{{- end }}
automountServiceAccountToken: false
containers:
- args:
- service-mirror
- -log-level={{.Values.localServiceMirror.logLevel}}
- -log-format={{.Values.localServiceMirror.logFormat}}
- -event-requeue-limit={{.Values.localServiceMirror.serviceMirrorRetryLimit}}
- -namespace={{.Release.Namespace}}
- -enable-pprof={{.Values.localServiceMirror.enablePprof | default false}}
- -local-mirror
- -federated-service-selector={{.Values.localServiceMirror.federatedServiceSelector}}
{{- if or .Values.localServiceMirror.additionalEnv .Values.localServiceMirror.experimentalEnv }}
env:
{{- with .Values.localServiceMirror.additionalEnv }}
{{- toYaml . | nindent 8 -}}
{{- end }}
{{- with .Values.localServiceMirror.experimentalEnv }}
{{- toYaml . | nindent 8 -}}
{{- end }}
{{- end }}
image: {{.Values.localServiceMirror.image.name}}:{{.Values.localServiceMirror.image.version}}
name: service-mirror
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: {{.Values.localServiceMirror.UID}}
runAsGroup: {{.Values.localServiceMirror.GID}}
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access
readOnly: true
ports:
- containerPort: 9999
name: admin-http
{{- with .Values.localServiceMirror.resources }}
resources: {{ toYaml . | nindent 10 }}
{{- end }}
securityContext:
seccompProfile:
type: RuntimeDefault
serviceAccountName: linkerd-local-service-mirror
volumes:
- {{- include "partials.volumes.manual-mount-service-account-token" . | indent 8 | trimPrefix (repeat 7 " ") }}
{{- with .Values.nodeSelector }}
nodeSelector: {{ toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations: {{ toYaml . | nindent 6 }}
{{- end }}
{{- if .Values.enablePodAntiAffinity }}
---
kind: PodDisruptionBudget
apiVersion: policy/v1
metadata:
name: linkerd-local-service-mirror
namespace: {{ .Release.Namespace }}
labels:
component: local-service-mirror
annotations:
{{ include "partials.annotations.created-by" . }}
spec:
maxUnavailable: 1
selector:
matchLabels:
component: local-service-mirror
{{- end}}
42 changes: 42 additions & 0 deletions multicluster/charts/linkerd-multicluster/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ gateway:

# -- Control plane version
linkerdVersion: linkerdVersionValue
# -- Additional annotations to add to all pods
podAnnotations: {}
# -- Additional labels to add to all pods
podLabels: {}
# -- Labels to apply to all resources
Expand Down Expand Up @@ -112,3 +114,43 @@ createNamespaceMetadataJob: true

# -- Specifies the number of old ReplicaSets to retain to allow rollback.
revisionHistoryLimit: 10

localServiceMirror:
# -- Number of times local service mirror updates are allowed to be requeued
# (retried)
serviceMirrorRetryLimit: 3

# -- Label selector for federated service members in the local cluster.
federatedServiceSelector: "mirror.linkerd.io/federated=member"

# -- Number of local service mirror replicas to run
replicas: 1

image:
# -- Docker image for the Service mirror component (uses the Linkerd controller
# image)
name: cr.l5d.io/linkerd/controller
# -- Pull policy for the Service mirror container image
# @default -- imagePullPolicy
pullPolicy: ""
# -- Tag for the Service mirror container image
# @default -- linkerdVersion
version: linkerdVersionValue

# -- Log level for the Multicluster components
logLevel: info

# -- Log format (`plain` or `json`)
logFormat: plain

# -- enables the use of pprof endpoints on control plane component's admin
# servers
enablePprof: false

# -- User id under which the Service Mirror shall be ran
UID: 2103
# -- Group id under which the Service Mirror shall be ran
GID: 2103

# -- Resources for the Service mirror container
resources: {}
2 changes: 2 additions & 0 deletions multicluster/cmd/install.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ var TemplatesMulticluster = []string{
"templates/remote-access-service-mirror-rbac.yaml",
"templates/link-crd.yaml",
"templates/service-mirror-policy.yaml",
"templates/local-service-mirror.yaml",
}

func newMulticlusterInstallCommand() *cobra.Command {
Expand Down Expand Up @@ -232,6 +233,7 @@ func buildMulticlusterInstallValues(ctx context.Context, opts *multiclusterInsta
return nil, err
}

defaults.LocalServiceMirror.Image.Version = version.Version
defaults.Gateway.Enabled = opts.gateway.Enabled
defaults.Gateway.Port = opts.gateway.Port
defaults.Gateway.Probe.Seconds = opts.gateway.Probe.Seconds
Expand Down
Loading

0 comments on commit 50b6a17

Please sign in to comment.