Skip to content

Commit

Permalink
iptables: use the same mode with kube-proxy (#2758)
Browse files Browse the repository at this point in the history
* iptables: use the same mode with kube-proxy (#2535)

* build base images for pr on necessary
  • Loading branch information
zhangzujian authored May 6, 2023
1 parent 944f304 commit 87c6868
Show file tree
Hide file tree
Showing 11 changed files with 532 additions and 114 deletions.
64 changes: 63 additions & 1 deletion .github/workflows/build-x86-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,46 @@ env:
HELM_VERSION: v3.11.1

jobs:
build-kube-ovn-base:
name: Build kube-ovn-base
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 2
- uses: docker/setup-buildx-action@v2
if: github.base_ref != null
- name: Build
run: |
touch .CI_PASSED_VAR
if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base$; then
echo "BUILD_BASE=1" > .CI_PASSED_VAR
echo "BUILD_BASE=1" >> "$GITHUB_ENV"
make base-amd64
make base-tar-amd64
fi
if git diff --name-only HEAD^ HEAD | grep -q ^dist/images/Dockerfile.base-dpdk$; then
make base-amd64-dpdk
fi
- name: Upload variable file to artifact
uses: actions/upload-artifact@v3
with:
name: variables
path: .CI_PASSED_VAR

- name: Upload base images to artifact
if: env.BUILD_BASE == 1
uses: actions/upload-artifact@v3
with:
name: kube-ovn-base
path: image-amd64.tar

build-kube-ovn:
name: Build kube-ovn
runs-on: ubuntu-22.04
needs:
- build-kube-ovn-base
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
Expand Down Expand Up @@ -62,12 +99,37 @@ jobs:
install "$tmp/gosec" /usr/local/bin
rm -rf $tmp
- name: Download variable file
uses: actions/download-artifact@v3
with:
name: variables

- name: Export passed variables
run: cat .CI_PASSED_VAR >> "$GITHUB_ENV"

- name: Download base images
if: env.BUILD_BASE == 1
uses: actions/download-artifact@v3
with:
name: kube-ovn-base

- name: Load base images
if: env.BUILD_BASE == 1
run: docker load --input image-amd64.tar

- name: Build
run: |
go mod tidy
git diff --exit-code
make lint
make image-kube-ovn
if [ "x${{ env.BUILD_BASE }}" = "x1" ]; then
TAG=$(cat VERSION)
docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
docker tag kubeovn/kube-ovn-base:$TAG-amd64-no-avx512 kubeovn/kube-ovn-base:$TAG-no-avx512
make build-kube-ovn
else
make image-kube-ovn
fi
make tar-kube-ovn
- name: Upload images to artifact
Expand Down
26 changes: 16 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,15 @@ build-go-arm:
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-cmd -ldflags $(GOLDFLAGS) -v ./cmd
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-webhook -ldflags $(GOLDFLAGS) -v ./cmd/webhook

.PHONY: build-kube-ovn
build-kube-ovn: build-go
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -f dist/images/Dockerfile dist/images/
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -f dist/images/Dockerfile.no-avx512 dist/images/
docker build -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -f dist/images/Dockerfile.dpdk dist/images/

.PHONY: build-dev
build-dev: build-go
docker build --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(DEV_TAG) -f dist/images/Dockerfile dist/images/
docker build -t $(REGISTRY)/kube-ovn:$(DEV_TAG) -f dist/images/Dockerfile dist/images/

.PHONY: build-dpdk
build-dpdk:
Expand All @@ -69,34 +75,34 @@ base-arm64:

.PHONY: image-kube-ovn
image-kube-ovn: build-go
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o type=docker -f dist/images/Dockerfile.no-avx512 dist/images/
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-no-avx512 -o type=docker -f dist/images/Dockerfile.no-avx512 dist/images/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/

.PHONY: image-debug
image-debug: build-go
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:debug -o type=docker -f dist/images/Dockerfile.debug dist/images/

.PHONY: image-vpc-nat-gateway
image-vpc-nat-gateway:
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway

.PHONY: image-centos-compile
image-centos-compile:
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
# docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
# docker buildx build --platform linux/amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/

.PHOONY: image-test
image-test: build-go
docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/test:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile.test dist/images/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/test:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile.test dist/images/

.PHONY: release
release: lint image-kube-ovn image-vpc-nat-gateway image-centos-compile

.PHONY: release-arm
release-arm: build-go-arm
docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
docker buildx build --platform linux/arm64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
docker buildx build --platform linux/arm64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway

.PHONY: push-dev
push-dev:
Expand Down
1 change: 1 addition & 0 deletions dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller

WORKDIR /kube-ovn

RUN /kube-ovn/iptables-wrapper-installer.sh --no-sanity-check
RUN rm -f /usr/bin/nc &&\
rm -f /usr/bin/netcat
RUN deluser sync
Expand Down
3 changes: 0 additions & 3 deletions dist/images/Dockerfile.base
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
logrotate dnsutils net-tools strongswan strongswan-pki libcharon-extra-plugins \
libcharon-extauth-plugins libstrongswan-extra-plugins libstrongswan-standard-plugins -y --no-install-recommends && \
rm -rf /var/lib/apt/lists/* && \
cd /usr/sbin && \
ln -sf /usr/sbin/iptables-legacy iptables && \
ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
rm -rf /etc/localtime

RUN mkdir -p /var/run/openvswitch && \
Expand Down
211 changes: 211 additions & 0 deletions dist/images/iptables-wrapper-installer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
#!/bin/sh

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Usage:
#
# iptables-wrapper-installer.sh [--no-sanity-check]
#
# Installs a wrapper iptables script in a container that will figure out
# whether iptables-legacy or iptables-nft is in use on the host and then
# replaces itself with the correct underlying iptables version.
#
# Unless "--no-sanity-check" is passed, it will first verify that the
# container already contains a suitable version of iptables.

# NOTE: This can only use POSIX /bin/sh features; the build container
# might not contain bash.

# original source:
# https://github.com/kubernetes-sigs/iptables-wrappers/blob/master/iptables-wrapper-installer.sh

set -eu

# Find iptables binary location
if [ -d /usr/sbin -a -e /usr/sbin/iptables ]; then
sbin="/usr/sbin"
elif [ -d /sbin -a -e /sbin/iptables ]; then
sbin="/sbin"
else
echo "ERROR: iptables is not present in either /usr/sbin or /sbin" 1>&2
exit 1
fi

# Determine how the system selects between iptables-legacy and iptables-nft
if [ -x /usr/sbin/alternatives ]; then
# Fedora/SUSE style alternatives
altstyle="fedora"
elif [ -x /usr/sbin/update-alternatives ]; then
# Debian style alternatives
altstyle="debian"
else
# No alternatives system
altstyle="none"
fi

if [ "${1:-}" != "--no-sanity-check" ]; then
# Ensure dependencies are installed
if ! version=$("${sbin}/iptables-nft" --version 2> /dev/null); then
echo "ERROR: iptables-nft is not installed" 1>&2
exit 1
fi
if ! "${sbin}/iptables-legacy" --version > /dev/null 2>&1; then
echo "ERROR: iptables-legacy is not installed" 1>&2
exit 1
fi

case "${version}" in
*v1.8.[0123]\ *)
echo "ERROR: iptables 1.8.0 - 1.8.3 have compatibility bugs." 1>&2
echo " Upgrade to 1.8.4 or newer." 1>&2
exit 1
;;
*)
# 1.8.4+ are OK
;;
esac
fi

# Start creating the wrapper...
rm -f "${sbin}/iptables-wrapper"
cat > "${sbin}/iptables-wrapper" <<EOF
#!/bin/sh
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: This can only use POSIX /bin/sh features; the container image
# might not contain bash.
set -eu
# In kubernetes 1.17 and later, kubelet will have created at least
# one chain in the "mangle" table (either "KUBE-IPTABLES-HINT" or
# "KUBE-KUBELET-CANARY"), so check that first, against
# iptables-nft, because we can check that more efficiently and
# it's more common these days.
nft_kubelet_rules=\$( (iptables-nft-save -t mangle || true; ip6tables-nft-save -t mangle || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${nft_kubelet_rules}" -ne 0 ]; then
mode=nft
else
# Check for kubernetes 1.17-or-later with iptables-legacy. We
# can't pass "-t mangle" to iptables-legacy-save because it would
# cause the kernel to create that table if it didn't already
# exist, which we don't want. So we have to grab all the rules
legacy_kubelet_rules=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep -E '^:(KUBE-IPTABLES-HINT|KUBE-KUBELET-CANARY)' | wc -l)
if [ "\${legacy_kubelet_rules}" -ne 0 ]; then
mode=legacy
else
# With older kubernetes releases there may not be any _specific_
# rules we can look for, but we assume that some non-containerized process
# (possibly kubelet) will have created _some_ iptables rules.
num_legacy_lines=\$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l)
num_nft_lines=\$( (iptables-nft-save || true; ip6tables-nft-save || true) 2>/dev/null | grep '^-' | wc -l)
if [ "\${num_legacy_lines}" -gt "\${num_nft_lines}" ]; then
mode=legacy
else
mode=nft
fi
fi
fi
EOF

# Write out the appropriate alternatives-selection commands
case "${altstyle}" in
fedora)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
EOF
;;

debian)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
update-alternatives --set iptables "/usr/sbin/iptables-\${mode}" > /dev/null || failed=1
update-alternatives --set ip6tables "/usr/sbin/ip6tables-\${mode}" > /dev/null || failed=1
EOF
;;

*)
cat >> "${sbin}/iptables-wrapper" <<EOF
# Update links to point to the selected binaries
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/\${cmd}"
ln -s "${sbin}/xtables-\${mode}-multi" "${sbin}/\${cmd}"
done 2>/dev/null || failed=1
EOF
;;
esac

# Write out the post-alternatives-selection error checking and final wrap-up
cat >> "${sbin}/iptables-wrapper" <<EOF
if [ "\${failed:-0}" = 1 ]; then
echo "Unable to redirect iptables binaries. (Are you running in an unprivileged pod?)" 1>&2
# fake it, though this will probably also fail if they aren't root
exec "${sbin}/xtables-\${mode}-multi" "\$0" "\$@"
fi
# Now re-exec the original command with the newly-selected alternative
exec "\$0" "\$@"
EOF
chmod +x "${sbin}/iptables-wrapper"

# Now back in the installer script, point the iptables binaries at our
# wrapper
case "${altstyle}" in
fedora)
alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables iptables /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save iptables-save /usr/sbin/iptables-wrapper
;;

debian)
update-alternatives \
--install /usr/sbin/iptables iptables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/iptables-restore iptables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/iptables-save iptables-save /usr/sbin/iptables-wrapper
update-alternatives \
--install /usr/sbin/ip6tables ip6tables /usr/sbin/iptables-wrapper 100 \
--slave /usr/sbin/ip6tables-restore ip6tables-restore /usr/sbin/iptables-wrapper \
--slave /usr/sbin/ip6tables-save ip6tables-save /usr/sbin/iptables-wrapper
;;

*)
for cmd in iptables iptables-save iptables-restore ip6tables ip6tables-save ip6tables-restore; do
rm -f "${sbin}/${cmd}"
ln -s "${sbin}/iptables-wrapper" "${sbin}/${cmd}"
done
;;
esac

# Cleanup
rm -f "$0"
3 changes: 3 additions & 0 deletions dist/images/start-cniserver.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ do
fi
done

# update links to point to the iptables binaries
iptables -V

# If nftables not exist do not exit
set +e
iptables -P FORWARD ACCEPT
Expand Down
3 changes: 3 additions & 0 deletions dist/images/start-ovs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ function quit {
}
trap quit EXIT

# update links to point to the iptables binaries
iptables -V

# Start ovsdb
/usr/share/openvswitch/scripts/ovs-ctl restart --no-ovs-vswitchd --system-id=random
# Restrict the number of pthreads ovs-vswitchd creates to reduce the
Expand Down
Loading

0 comments on commit 87c6868

Please sign in to comment.