-
Notifications
You must be signed in to change notification settings - Fork 208
377 lines (329 loc) · 15.5 KB
/
kind.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
name: Kind
# Any change in triggers needs to be reflected in the concurrency group.
on:
pull_request: {}
schedule:
- cron: '30 */2 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || 'scheduled' }}
cancel-in-progress: true
env:
# renovate: datasource=github-releases depName=kubernetes-sigs/kind
kind_version: v0.20.0
kind_config: .github/kind-config.yaml
TIMEOUT: 2m
LOG_TIME: 30m
# renovate: datasource=github-releases depName=cilium/cilium
cilium_version: v1.14.2
kubectl_version: v1.23.6
jobs:
installation-and-connectivity:
runs-on: ubuntu-22.04
timeout-minutes: 50
strategy:
matrix:
mode: ["classic", "helm"]
steps:
- name: Set mode
run: |
echo "CILIUM_CLI_MODE=${{ matrix.mode }}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Install kubectl
run: |
curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl"
curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum --check
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
- name: Set up Go
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
# renovate: datasource=golang-version depName=go
go-version: 1.21.3
- name: Set up Go for root
run: |
sudo ln -sf `which go` `sudo which go` || true
sudo go version
- name: Build and install cilium CLI binary
run: sudo make install
- name: Create kind cluster
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
with:
version: ${{ env.kind_version }}
config: ${{ env.kind_config }}
- name: Set NODES_WITHOUT_CILIUM
run: |
# To add more elements, keep it comma-separated.
echo "NODES_WITHOUT_CILIUM=chart-testing-worker2,chart-testing-worker3" >> $GITHUB_ENV
# Install Cilium with HostPort support and enables Prometheus for extended connectivity test.
- name: Install Cilium
run: |
cilium install \
--version=${{ env.cilium_version }} \
--nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \
--wait=false \
--set bpf.monitorAggregation=none \
--set cni.chainingMode=portmap \
--set loadBalancer.l7.backend=envoy \
--set tls.secretsBackend=k8s \
--set prometheus.enabled=true
- name: Enable Relay
run: |
cilium hubble enable --ui
cilium status --wait
- name: Relay Port Forward
run: |
cilium hubble port-forward&
sleep 10s
[[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
- name: Set up external targets
id: external_targets
run: |
export worker2_ip=$(kubectl get nodes chart-testing-worker2 -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
export worker3_ip=$(kubectl get nodes chart-testing-worker3 -o=jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
echo "worker2_ip=${worker2_ip}" >> $GITHUB_OUTPUT
echo "worker3_ip=${worker3_ip}" >> $GITHUB_OUTPUT
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
kubectl rollout status -n cert-manager deployment.apps/cert-manager
kubectl rollout status -n cert-manager deployment.apps/cert-manager-webhook
kubectl create ns external-targets
cat .github/external-targets/certs.yaml | envsubst | kubectl apply -n external-targets -f -
until kubectl get secret -n external-targets external-target-cert &> /dev/null; do sleep 1s; done
kubectl apply -n external-targets -f .github/external-targets/nginx.yaml
kubectl rollout status -n external-targets ds/nginx
- name: Connectivity Test
run: |
# Setup the connectivity disruption tests. We don't really care about the result
# here (in the sense that we don't perform any operation which might cause a
# disruption), but we want to make sure that the command works as expected.
cilium connectivity test --debug --test-namespace test-namespace \
--include-conn-disrupt-test --conn-disrupt-test-setup
# Run the connectivity test in non-default namespace (i.e. not cilium-test)
cilium connectivity test --debug --all-flows --test-namespace test-namespace \
--include-unsafe-tests --include-conn-disrupt-test \
--collect-sysdump-on-failure --junit-file cilium-junit-${{ matrix.mode }}-1.xml \
--junit-property mode=${{ matrix.mode }} --junit-property type=no-tunnel \
--curl-insecure \
--external-target chart-testing-worker2 \
--external-target-ca-namespace=external-targets \
--external-target-ca-name=ca \
--external-cidr 172.18.0.0/16 \
--external-ip ${{ steps.external_targets.outputs.worker2_ip }} \
--external-other-ip ${{ steps.external_targets.outputs.worker3_ip }}
- name: Uninstall cilium
run: |
pkill -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" || test $? -eq 1
cilium uninstall --wait
- name: Install Cilium with IPsec Encryption
if: ${{ matrix.mode == 'classic' }}
run: |
cilium install \
--version=${{ env.cilium_version}} \
--encryption=ipsec \
--nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \
--set kubeProxyReplacement=disabled
- name: Install Cilium with IPsec Encryption
if: ${{ matrix.mode == 'helm' }}
run: |
kubectl create -n kube-system secret generic cilium-ipsec-keys \
--from-literal=keys="3 rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"
cilium install \
--version=${{ env.cilium_version}} \
--nodes-without-cilium="${NODES_WITHOUT_CILIUM}" \
--set encryption.enabled=true \
--set encryption.type=ipsec \
--set kubeProxyReplacement=disabled
- name: Enable Relay
run: |
cilium hubble enable
cilium status --wait
- name: Relay Port Forward
run: |
cilium hubble port-forward&
sleep 10s
[[ $(pgrep -f "cilium.*hubble.*port-forward|kubectl.*port-forward.*hubble-relay" | wc -l) == 2 ]]
- name: Connectivity test
run: |
cilium connectivity test --debug --force-deploy --all-flows --test-namespace test-namespace \
--include-unsafe-tests \
--collect-sysdump-on-failure --junit-file cilium-junit-${{ matrix.mode }}-2.xml \
--junit-property mode=${{ matrix.mode }} --junit-property type=ipsec \
--curl-insecure \
--external-target chart-testing-worker2 \
--external-target-ca-namespace=external-targets \
--external-target-ca-name=ca \
--external-cidr 172.18.0.0/16 \
--external-ip ${{ steps.external_targets.outputs.worker2_ip }} \
--external-other-ip ${{ steps.external_targets.outputs.worker3_ip }}
- name: Upload JUnit
if: ${{ always() }}
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: cilium-junits
path: cilium-junit*.xml
retention-days: 2
- name: Cleanup
if: ${{ always() }}
run: |
cilium status
kubectl get pods --all-namespaces -o wide
cilium sysdump --output-filename cilium-sysdump-out --hubble-flows-count 10000
shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
- name: Unlabel nodes
if: ${{ matrix.mode == 'helm' }}
run: |
IFS=',' read -ra nodes <<< "$NODES_WITHOUT_CILIUM"
for node in "${nodes[@]}"; do
kubectl label nodes "${node}" cilium.io/no-schedule-
done
- name: Upload sysdump
if: ${{ !success() }}
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: cilium-sysdumps
path: cilium-sysdump-*.zip
retention-days: 5
helm-upgrade-clustermesh:
runs-on: ubuntu-22.04
timeout-minutes: 50
env:
CILIUM_CLI_MODE: helm
kind_config_1: .github/kind-config-1.yaml
kind_config_2: .github/kind-config-2.yaml
# helm/kind-action will override the "name:" provided in the kind config with "chart-testing" unless these are
# specified as inputs. These must also match the suffix here for CLUSTER1 and CLUSTER2.
CLUSTER_NAME_1: c.1
CLUSTER_NAME_2: c.2
CLUSTER1: kind-c.1
CLUSTER2: kind-c.2
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Install kubectl
run: |
curl -sLO "https://dl.k8s.io/release/${{ env.kubectl_version }}/bin/linux/amd64/kubectl"
curl -sLO "https://dl.k8s.io/${{ env.kubectl_version }}/bin/linux/amd64/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum --check
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
- name: Set up Go
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
# renovate: datasource=golang-version depName=go
go-version: 1.21.3
- name: Set up Go for root
run: |
sudo ln -sf `which go` `sudo which go` || true
sudo go version
- name: Build and install cilium CLI binary
run: sudo make install
- name: Create kind cluster 1
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
with:
version: ${{ env.kind_version }}
config: ${{ env.kind_config_1 }}
cluster_name: ${{ env.CLUSTER_NAME_1 }}
- name: Install Cilium on cluster 1
run: |
cilium install --context $CLUSTER1 \
--version=${{ env.cilium_version }} \
--wait=true \
--set bpf.monitorAggregation=none \
--set cni.chainingMode=portmap \
--set cluster.id=1 \
--set cluster.name=$CLUSTER1
- name: Create kind cluster 2
uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0
with:
version: ${{ env.kind_version }}
config: ${{ env.kind_config_2 }}
cluster_name: ${{ env.CLUSTER_NAME_2 }}
- name: Install Cilium on cluster 2
run: |
cilium install --context $CLUSTER2 \
--version=${{ env.cilium_version }} \
--wait=true \
--set bpf.monitorAggregation=none \
--set cni.chainingMode=portmap \
--set cluster.id=2 \
--set cluster.name=$CLUSTER2
- name: Enable ClusterMesh on cluster 1 using helm-based upgrade
run: |
cilium upgrade --reuse-values --context $CLUSTER1 \
--wait=true \
--set clustermesh.useAPIServer=true \
--set clustermesh.apiserver.service.type=NodePort \
--set clustermesh.apiserver.tls.server.extraDnsNames={"$CLUSTER1.mesh.cilium.io,$CLUSTER2.mesh.cilium.io"}
- name: Copy CA cert from cluster 1 to cluster 2
run: |
kubectl --context $CLUSTER2 delete secret -n kube-system cilium-ca && \
kubectl --context $CLUSTER1 get secrets -n kube-system cilium-ca -oyaml \
| kubectl --context $CLUSTER2 apply -f -
# Restart Cilium on cluster 2
kubectl --context $CLUSTER2 delete pod -l app.kubernetes.io/part-of=cilium -A
- name: Enable ClusterMesh on cluster 2 using helm-based upgrade
run: |
cilium upgrade --reuse-values --context $CLUSTER2 \
--wait=true \
--set clustermesh.useAPIServer=true \
--set clustermesh.apiserver.service.type=NodePort \
--set clustermesh.apiserver.tls.server.extraDnsNames={"$CLUSTER1.mesh.cilium.io,$CLUSTER2.mesh.cilium.io"}
- name: Rename the secrets expected by the clustermesh connect command
run: |
kubectl get secrets --context $CLUSTER1 \
-n kube-system clustermesh-apiserver-remote-cert -oyaml \
| sed 's/name: .*/name: clustermesh-apiserver-client-cert/' \
| kubectl apply --context $CLUSTER1 -f -
kubectl get secrets --context $CLUSTER2 \
-n kube-system clustermesh-apiserver-remote-cert -oyaml \
| sed 's/name: .*/name: clustermesh-apiserver-client-cert/' \
| kubectl apply --context $CLUSTER2 -f -
- name: Connect the two clusters using clustermesh
run: |
cilium clustermesh connect --context $CLUSTER1 --destination-context $CLUSTER2
cilium clustermesh status --context $CLUSTER1 --wait
- name: Run the multicluster connectivity tests
run: |
# Remove unused docker stuff to avoid running out of disk space.
docker system prune -fa
# Setup the connectivity disruption tests. We don't really care about the result
# here (in the sense that we don't perform any operation which might cause a
# disruption), but we want to make sure that the command works as expected.
cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug \
--include-conn-disrupt-test --conn-disrupt-test-setup
cilium connectivity test --context $CLUSTER1 --multi-cluster $CLUSTER2 --debug \
--include-unsafe-tests --include-conn-disrupt-test \
--collect-sysdump-on-failure --junit-file cilium-junit-clustermesh-1.xml \
--junit-property mode=clustermesh --junit-property type=ipsec
- name: Upload JUnit
if: ${{ always() }}
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: cilium-junits
path: cilium-junit*.xml
retention-days: 2
- name: Cleanup
if: ${{ always() }}
run: |
cilium --context $CLUSTER1 status
kubectl --context $CLUSTER1 get pods --all-namespaces -o wide
cilium --context $CLUSTER1 sysdump --output-filename cilium-sysdump-out-c1
cilium --context $CLUSTER2 status
kubectl --context $CLUSTER2 get pods --all-namespaces -o wide
cilium --context $CLUSTER2 sysdump --output-filename cilium-sysdump-out-c2
shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently
- name: Upload sysdump from cluster 1
if: ${{ !success() }}
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: cilium-sysdump-out-c1.zip
path: cilium-sysdump-out-c1.zip
retention-days: 5
- name: Upload sysdump from cluster 2
if: ${{ !success() }}
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
with:
name: cilium-sysdump-out-c2.zip
path: cilium-sysdump-out-c2.zip
retention-days: 5