Skip to content

Commit

Permalink
Update MetalLB deployment, wait for resource. (kubernetes-sigs#9995)
Browse files Browse the repository at this point in the history
* Update MetalLB deployment, wait for resource.

Signed-off-by: Jeroen Rijken <[email protected]>

* yml to yaml, add basic test for metallb

Signed-off-by: Jeroen Rijken <[email protected]>

---------

Signed-off-by: Jeroen Rijken <[email protected]>
  • Loading branch information
Jeroen0494 authored and pedromcpedro committed May 8, 2024
1 parent 77f3cd4 commit 277902c
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 84 deletions.
73 changes: 48 additions & 25 deletions docs/metallb.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,39 @@ By default only the MetalLB BGP speaker is allowed to run on control plane nodes
```yaml
metallb_config:
controller:
nodeselector:
kubernetes.io/os: linux
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Equal"
value: ""
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
value: ""
effect: "NoSchedule"
- key: "node-role.kubernetes.io/master"
operator: "Equal"
value: ""
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
value: ""
effect: "NoSchedule"
```
If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:
```yaml
metallb_config:
controller:
nodeselector:
kubernetes.io/os: linux
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
value: ""
effect: "NoSchedule"
speaker:
nodeselector:
kubernetes.io/os: linux
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
value: ""
effect: "NoSchedule"
```
## Pools
Expand Down Expand Up @@ -137,7 +161,6 @@ In this scenario you should disable the MetalLB speaker and configure the `calic

```yaml
metallb_speaker_enabled: false
metallb_avoid_buggy_ips: true
metallb_config:
address_pools:
primary:
Expand Down Expand Up @@ -177,22 +200,22 @@ metallb_config:
vpn-only: "1234:1"
NO_ADVERTISE: "65535:65282"
metallb_peers:
peer1:
peer_address: 10.6.0.1
peer_asn: 64512
my_asn: 4200000000
communities:
- vpn-only
address_pool:
- pool1
peer2:
peer_address: 10.10.0.1
peer_asn: 64513
my_asn: 4200000000
communities:
- NO_ADVERTISE
address_pool:
- pool2
peer1:
peer_address: 10.6.0.1
peer_asn: 64512
my_asn: 4200000000
communities:
- vpn-only
address_pool:
- pool1
peer2:
peer_address: 10.10.0.1
peer_asn: 64513
my_asn: 4200000000
communities:
- NO_ADVERTISE
address_pool:
- pool2
calico_advertise_service_loadbalancer_ips:
- 10.5.0.0/16
- 10.6.0.0/16
Expand Down
5 changes: 0 additions & 5 deletions roles/kubernetes-apps/metallb/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
---
metallb_enabled: false
metallb_log_level: info
metallb_protocol: "layer2"
metallb_port: "7472"
metallb_memberlist_port: "7946"
metallb_peers: []
metallb_speaker_enabled: "{{ metallb_enabled }}"
metallb_speaker_nodeselector:
kubernetes.io/os: "linux"
Expand All @@ -18,6 +16,3 @@ metallb_speaker_tolerations:
key: node-role.kubernetes.io/control-plane
operator: Exists
metallb_controller_tolerations: []
metallb_pool_name: "loadbalanced"
metallb_auto_assign: true
metallb_avoid_buggy_ips: false
114 changes: 78 additions & 36 deletions roles/kubernetes-apps/metallb/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,6 @@
when:
- "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"

- name: Kubernetes Apps | Check BGP peers for MetalLB
fail:
msg: "metallb_peers is mandatory when metallb_protocol is bgp and metallb_speaker_enabled"
when:
- metallb_config.layer3 is defined and metallb_speaker_enabled
- metallb_config.metallb_peers is not defined or not metallb_config.metallb_peers

- name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore
fail:
msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead"
Expand All @@ -36,46 +29,95 @@
- name: Kubernetes Apps | Lay Down MetalLB
become: true
template:
src: "{{ item }}.j2"
dest: "{{ kube_config_dir }}/{{ item }}"
src: "metallb.yaml.j2"
dest: "{{ kube_config_dir }}/metallb.yaml"
mode: 0644
with_items: ["metallb.yml", "pools.yaml", "layer2.yaml", "layer3.yaml"]
register: "rendering"
register: metallb_rendering
when:
- "inventory_hostname == groups['kube_control_plane'][0]"

- name: Kubernetes Apps | Create MetalLB resources and replace existing
k8s:
definition: "{{ lookup('template', 'metallb.yaml') }}"

- name: Kubernetes Apps | Wait for MetalLB controller to be running
k8s_info:
kind: Deployment
namespace: metallb-system
name: controller
wait: True
wait_sleep: 10
wait_timeout: 360
wait_condition:
status: "True"
type: Available
register: result
until: result is not failed
- inventory_hostname == groups['kube_control_plane'][0]

- name: Kubernetes Apps | Install and configure MetalLB
kube:
name: "MetalLB"
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/{{ item.item }}"
state: "{{ item.changed | ternary('latest','present') }}"
filename: "{{ kube_config_dir }}/metallb.yaml"
state: "{{ metallb_rendering.changed | ternary('latest','present') }}"
wait: true
become: true
with_items: "{{ rendering.results }}"
when:
- "inventory_hostname == groups['kube_control_plane'][0]"
- inventory_hostname == groups['kube_control_plane'][0]

- name: Kubernetes Apps | Wait for MetalLB controller to be running
command: "{{ bin_dir }}/kubectl -n metallb-system wait --for=condition=ready pod -l app=metallb,component=controller"
become: true
when:
- inventory_hostname == groups['kube_control_plane'][0]

- name: MetalLB | Address pools
block:
- name: MetalLB | Layout address pools template
ansible.builtin.template:
src: pools.yaml.j2
dest: "{{ kube_config_dir }}/pools.yaml"
mode: 0644
register: pools_rendering

- name: MetalLB | Create address pools configuration
kube:
name: "MetalLB"
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/pools.yaml"
state: "{{ pools_rendering.changed | ternary('latest','present') }}"
become: true
when:
- inventory_hostname == groups['kube_control_plane'][0]
- metallb_config.address_pools is defined

- name: MetalLB | Layer2
block:
- name: MetalLB | Layout layer2 template
ansible.builtin.template:
src: layer2.yaml.j2
dest: "{{ kube_config_dir }}/layer2.yaml"
mode: 0644
register: layer2_rendering

- name: MetalLB | Create layer2 configuration
kube:
name: "MetalLB"
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/layer2.yaml"
state: "{{ layer2_rendering.changed | ternary('latest','present') }}"
become: true
when:
- inventory_hostname == groups['kube_control_plane'][0]
- metallb_config.layer2 is defined

- name: MetalLB | Layer3
block:
- name: MetalLB | Layout layer3 template
ansible.builtin.template:
src: layer3.yaml.j2
dest: "{{ kube_config_dir }}/layer3.yaml"
mode: 0644
register: layer3_rendering

- name: MetalLB | Create layer3 configuration
kube:
name: "MetalLB"
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/layer3.yaml"
state: "{{ layer3_rendering.changed | ternary('latest','present') }}"
become: true
when:
- inventory_hostname == groups['kube_control_plane'][0]
- metallb_config.layer3 is defined


- name: Kubernetes Apps | Delete MetalLB ConfigMap
k8s:
kube:
name: config
kind: ConfigMap
kubectl: "{{ bin_dir }}/kubectl"
resource: ConfigMap
namespace: metallb-system
state: absent
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
---
apiVersion: v1
kind: Namespace
metadata:
labels:
pod-security.kubernetes.io/audit: privileged
pod-security.kubernetes.io/enforce: privileged
pod-security.kubernetes.io/warn: privileged
name: metallb-system

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -1703,8 +1713,8 @@ spec:
template:
metadata:
annotations:
prometheus.io/port: "{{ metallb_port }}"
prometheus.io/scrape: "true"
prometheus.io/port: '{{ metallb_port }}'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: controller
Expand All @@ -1719,7 +1729,7 @@ spec:
value: memberlist
- name: METALLB_DEPLOYMENT
value: controller
image: {{ metallb_controller_image_repo }}:{{ metallb_version }}
image: "{{ metallb_controller_image_repo }}:{{ metallb_version }}"
livenessProbe:
failureThreshold: 3
httpGet:
Expand Down Expand Up @@ -1755,14 +1765,15 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
{% if metallb_config.controller.tolerations %}
{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %}
tolerations:
{{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
{% endif %}
{% if metallb_controller_nodeselector %}
nodeSelector:
{{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) }}
{%- endif %}
nodeSelector:
{{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}}
{% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %}
{{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}}
{%- endif %}
securityContext:
fsGroup: 65534
runAsNonRoot: true
Expand Down Expand Up @@ -1793,8 +1804,8 @@ spec:
template:
metadata:
annotations:
prometheus.io/port: "{{ metallb_port }}"
prometheus.io/scrape: "true"
prometheus.io/port: '{{ metallb_port }}'
prometheus.io/scrape: 'true'
labels:
app: metallb
component: speaker
Expand Down Expand Up @@ -1823,7 +1834,7 @@ spec:
secretKeyRef:
key: secretkey
name: memberlist
image: {{ metallb_speaker_image_repo }}:{{ metallb_version }}
image: "{{ metallb_speaker_image_repo }}:{{ metallb_version }}"
livenessProbe:
failureThreshold: 3
httpGet:
Expand Down Expand Up @@ -1860,16 +1871,19 @@ spec:
- ALL
readOnlyRootFilesystem: true
hostNetwork: true
{% if metallb_speaker_nodeselector %}
nodeSelector:
{{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) }}
{%- endif %}
{{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}}
{% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %}
{{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}}
{%- endif %}

serviceAccountName: speaker
terminationGracePeriodSeconds: 2
{% if metallb_speaker_tolerations %}
tolerations:
{{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
{% endif %}
{{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
{% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %}
{{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
{% endif %}
{% endif %}

---
Expand Down Expand Up @@ -2004,7 +2018,7 @@ webhooks:
clientConfig:
service:
name: webhook-service
namespace: metallb-system
namespace: metallb-system
path: /validate-metallb-io-v1beta1-l2advertisement
failurePolicy: Fail
name: l2advertisementvalidationwebhook.metallb.io
Expand Down
23 changes: 23 additions & 0 deletions tests/files/packet_centos7-flannel-addons-ha.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,26 @@ kube_vip_enabled: true
kube_vip_arp_enabled: true
kube_vip_controlplane_enabled: true
kube_vip_address: 192.168.1.100

# MetalLB
metallb_enabled: true
metallb_speaker_enabled: true
metallb_config:
address_pools:
primary:
ip_range:
- 192.0.1.0-192.0.1.254
auto_assign: true
pool1:
ip_range:
- 192.0.2.1-192.0.2.1
auto_assign: false
pool2:
ip_range:
- 192.0.2.2-192.0.2.2
auto_assign: false

layer2:
- primary
- pool1
- pool2

0 comments on commit 277902c

Please sign in to comment.