Skip to content

Commit

Permalink
feat: Allow running images without explicit metadata labels (#2582)
Browse files Browse the repository at this point in the history 
Co-authored-by: Joongi Kim <[email protected]>
  • Loading branch information
@kyujin-cho @achimnol
kyujin-cho and achimnol authored Aug 8, 2024
1 parent 045311f commit 163c811
Show file tree
Hide file tree
Showing 109 changed files with 324 additions and 591 deletions.
1 change: 1 addition & 0 deletions changes/2582.feature.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Now Backend.AI can run arbitrary container images without Backend.AI-specific metadata labels by introducing good default values and replacing intrinsic kernel-runner binaries with statically built ones
1 change: 1 addition & 0 deletions scripts/agent/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*-builder*.dockerfile
117 changes: 43 additions & 74 deletions scripts/agent/build-dropbear.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,69 +2,50 @@
set -e

arch=$(uname -m)
distros=("alpine3.8" "centos8.0" "ubuntu18.04" "ubuntu20.04" "ubuntu22.04")
if [ $arch = "arm64" ]; then
arch="aarch64"
fi

ubuntu1804_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:18.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf automake zlib1g-dev
EOF
)
ubuntu2004_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:20.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf automake zlib1g-dev
EOF
)
ubuntu2204_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:22.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf automake zlib1g-dev
EOF
)
alpine_builder_dockerfile=$(cat <<'EOF'
FROM alpine:3.8
RUN apk add --no-cache make gcc musl-dev
RUN apk add --no-cache autoconf automake zlib-dev
EOF
)
centos8_builder_dockerfile=$(cat <<'EOF'
FROM centos:centos8
RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*
RUN dnf install -y make gcc automake autoconf dnf-plugins-core
RUN dnf config-manager --set-enabled powertools
RUN dnf install -y zlib-static glibc-static libxcrypt-static
builder_dockerfile=$(cat <<'EOF'
FROM alpine:3.20
RUN apk add --no-cache make gcc musl-dev autoconf automake git wget
RUN apk add --no-cache zlib-dev zlib-static libtool pkgconfig
RUN wget https://ftp.gnu.org/gnu/shtool/shtool-2.0.8.tar.gz \
&& tar -xzf shtool-2.0.8.tar.gz \
&& cd shtool-2.0.8 \
&& ./configure && make && make install
RUN mkdir -p /opt && ln -s /usr/local/bin/shtool /opt/
EOF
)


build_script=$(cat <<'EOF'
#! /bin/sh
set -e
git clone -c advice.detachedHead=false --depth=1 \
--branch "DROPBEAR_2024.85" \
https://github.com/mkj/dropbear \
dropbear
cd dropbear
autoreconf
autoconf && autoheader
./configure --enable-static --prefix=/opt/kernel
# Improve SFTP up/download throughputs.
# FIXME: Temporarily falling back to the default to avoid PyCharm compatibility issue
sed -i 's/\(DEFAULT_RECV_WINDOW\) [0-9][0-9]*/\1 2097152/' default_options.h
sed -i 's/\(RECV_MAX_PAYLOAD_LEN\) [0-9][0-9]*/\1 2621440/' default_options.h
sed -i 's/\(TRANS_MAX_PAYLOAD_LEN\) [0-9][0-9]*/\1 2621440/' default_options.h
sed -i 's/\(MAX_CMD_LEN\) [0-9][0-9]*/\1 20000/' sysoptions.h
sed -i '/channel->transwindow -= len;/s/^/\/\//' common-channel.c
sed -i 's/DEFAULT_PATH/getenv("PATH")/' svr-chansession.c
sed -i 's/\(DEFAULT_RECV_WINDOW\) [0-9][0-9]*/\1 2097152/' src/default_options.h
sed -i 's/\(RECV_MAX_PAYLOAD_LEN\) [0-9][0-9]*/\1 2621440/' src/default_options.h
sed -i 's/\(TRANS_MAX_PAYLOAD_LEN\) [0-9][0-9]*/\1 2621440/' src/default_options.h
sed -i 's/\(MAX_CMD_LEN\) [0-9][0-9]*/\1 20000/' src/sysoptions.h
sed -i '/channel->transwindow -= len;/s/^/\/\//' src/common-channel.c
sed -i 's/DEFAULT_PATH/getenv("PATH")/' src/svr-chansession.c
# Disable clearing environment variables for new pty sessions and remote commands
sed -i 's%/\* *#define \+DEBUG_VALGRIND *\*/%#define DEBUG_VALGRIND%' debug.h
sed -i 's%/\* *#define \+DEBUG_VALGRIND *\*/%#define DEBUG_VALGRIND%' src/debug.h
make
cp dropbear ../dropbear.$X_DISTRO.$X_ARCH.bin
cp dropbearkey ../dropbearkey.$X_DISTRO.$X_ARCH.bin
cp dropbearconvert ../dropbearconvert.$X_DISTRO.$X_ARCH.bin
make -j$(nproc)
cp dropbear ../dropbear.$X_ARCH.bin
cp dropbearkey ../dropbearkey.$X_ARCH.bin
cp dropbearconvert ../dropbearconvert.$X_ARCH.bin
make clean
EOF
)
Expand All @@ -74,34 +55,22 @@ temp_dir=$(mktemp -d -t dropbear-build.XXXXX)
echo "Using temp directory: $temp_dir"
echo "$build_script" > "$temp_dir/build.sh"
chmod +x $temp_dir/*.sh
echo "$ubuntu1804_builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.ubuntu18.04.dockerfile"
echo "$ubuntu2004_builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.ubuntu20.04.dockerfile"
echo "$ubuntu2204_builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.ubuntu22.04.dockerfile"
echo "$alpine_builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.alpine3.8.dockerfile"
echo "$centos8_builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.centos8.0.dockerfile"

for distro in "${distros[@]}"; do
docker build -t dropbear-builder:$distro \
-f $SCRIPT_DIR/dropbear-builder.$distro.dockerfile $SCRIPT_DIR
done
echo "$builder_dockerfile" > "$SCRIPT_DIR/dropbear-builder.dockerfile"

cd "$temp_dir"
git clone -c advice.detachedHead=false --branch "DROPBEAR_2020.81" https://github.com/mkj/dropbear dropbear
docker build -t dropbear-builder \
-f $SCRIPT_DIR/dropbear-builder.dockerfile $SCRIPT_DIR

for distro in "${distros[@]}"; do
docker run --rm -it \
-e X_DISTRO=$distro \
-e X_ARCH=$arch \
-u $(id -u):$(id -g) \
-w /workspace \
-v $temp_dir:/workspace \
dropbear-builder:$distro \
/workspace/build.sh
done
docker run --rm -it \
-e X_ARCH=$arch \
-u $(id -u):$(id -g) \
-w /workspace \
-v $temp_dir:/workspace \
dropbear-builder \
/workspace/build.sh

ls -l .
cp dropbear.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp dropbearkey.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp dropbearconvert.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp $temp_dir/dropbear.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp $temp_dir/dropbearkey.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp $temp_dir/dropbearconvert.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
ls -lh src/ai/backend/runner

rm -rf "$temp_dir"
143 changes: 28 additions & 115 deletions scripts/agent/build-sftpserver.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,109 +2,36 @@
set -e

arch=$(uname -m)
distros=("alpine3.8" "centos7.6" "centos8.0" "ubuntu16.04" "ubuntu18.04" "ubuntu20.04")

static_libs_dockerfile_part=$(cat <<'EOF'
ENV ZLIB_VER=1.3.1 \
SSL_VER=1.1.1i
RUN wget https://www.zlib.net/zlib-${ZLIB_VER}.tar.gz -O /root/zlib-${ZLIB_VER}.tar.gz && \
wget https://www.openssl.org/source/openssl-${SSL_VER}.tar.gz -O /root/openssl-${SSL_VER}.tar.gz
RUN cd /root && \
tar xzvf zlib-${ZLIB_VER}.tar.gz && \
tar xzvf openssl-${SSL_VER}.tar.gz
RUN echo "BUILD: zlib" && \
cd /root/zlib-${ZLIB_VER} && \
./configure --prefix=/usr/local --static && \
make && \
make install
RUN echo "BUILD: OpenSSL" && \
cd /root/openssl-${SSL_VER} && \
./config --prefix=/usr no-shared --openssldir=/usr/local/openssl && \
make && \
make install
EOF
)

ubuntu1604_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:16.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf
RUN apt-get install -y wget
# below required for sys/mman.h
RUN apt-get install -y libc6-dev
EOF
)

ubuntu1804_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:18.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf
RUN apt-get install -y wget
# below required for sys/mman.h
RUN apt-get install -y libc6-dev
EOF
)

ubuntu2004_builder_dockerfile=$(cat <<'EOF'
FROM ubuntu:20.04
RUN apt-get update
RUN apt-get install -y make gcc
RUN apt-get install -y autoconf
RUN apt-get install -y wget
# below required for sys/mman.h
RUN apt-get install -y libc6-dev
EOF
)

centos_builder_dockerfile=$(cat <<'EOF'
FROM centos:7
RUN yum install -y make gcc
RUN yum install -y autoconf
RUN yum install -y wget
EOF
)

centos8_builder_dockerfile=$(cat <<'EOF'
FROM centos:centos8
RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-*
RUN dnf install -y make gcc
RUN dnf install -y autoconf
RUN dnf install -y wget
EOF
)

alpine_builder_dockerfile=$(cat <<'EOF'
FROM alpine:3.8
RUN apk add --no-cache make gcc musl-dev
RUN apk add --no-cache autoconf
RUN apk add --no-cache wget
if [ $arch = "arm64" ]; then
arch="aarch64"
fi

builder_dockerfile=$(cat <<'EOF'
FROM alpine:3.20
RUN apk add --no-cache make gcc musl-dev autoconf automake git wget
RUN apk add --no-cache zlib-dev zlib-static libressl-dev
# below required for sys/mman.h
RUN apk add --no-cache linux-headers
EOF
)

build_script=$(cat <<'EOF'
#! /bin/sh
echo "BUILD: OpenSSH"
cd /workspace/openssh-portable
set -e
git clone -c advice.detachedHead=false --depth=1 \
--branch "V_9_8_P1" \
https://github.com/openssh/openssh-portable \
openssh-portable
cd openssh-portable
autoreconf
export LDFLAGS="-L/root/zlib-${ZLIB_VER} -L/root/openssl-${SSL_VER} -pthread"
export LIBS="-ldl"
sed -i "s/-lcrypto/-l:libcrypto.a/" ./configure
sed -i "s/-lz/-l:libz.a/" ./configure
./configure --prefix=/usr
./configure --prefix=/usr --enable-static --with-ldflags=-static
sed -i 's/^# \?define SFTP_MAX_MSG_LENGTH[ \t]*.*/#define SFTP_MAX_MSG_LENGTH 5242880/g' sftp-common.h
make sftp-server scp
cp sftp-server ../sftp-server.$X_DISTRO.$X_ARCH.bin
cp scp ../scp.$X_DISTRO.$X_ARCH.bin
make clean
make -j$(nproc) sftp-server scp
cp sftp-server /workspace/sftp-server.$X_ARCH.bin
cp scp /workspace/scp.$X_ARCH.bin
EOF
)

Expand All @@ -113,35 +40,21 @@ temp_dir=$(mktemp -d -t sftpserver-build.XXXXX)
echo "Using temp directory: $temp_dir"
echo "$build_script" > "$temp_dir/build.sh"
chmod +x $temp_dir/*.sh
echo -e "$ubuntu1604_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.ubuntu16.04.dockerfile"
echo -e "$ubuntu1804_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.ubuntu18.04.dockerfile"
echo -e "$ubuntu2004_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.ubuntu20.04.dockerfile"
echo -e "$centos_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.centos7.6.dockerfile"
echo -e "$centos8_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.centos8.0.dockerfile"
echo -e "$alpine_builder_dockerfile\n$static_libs_dockerfile_part" > "$SCRIPT_DIR/sftpserver-builder.alpine3.8.dockerfile"

for distro in "${distros[@]}"; do
docker build -t sftpserver-builder:$distro \
-f $SCRIPT_DIR/sftpserver-builder.$distro.dockerfile $SCRIPT_DIR
done
echo -e "$builder_dockerfile" > "$SCRIPT_DIR/sftpserver-builder.dockerfile"

cd "$temp_dir"
git clone -c advice.detachedHead=false --branch "V_8_1_P1" https://github.com/openssh/openssh-portable openssh-portable
docker build -t sftpserver-builder -f $SCRIPT_DIR/sftpserver-builder.dockerfile $SCRIPT_DIR

for distro in "${distros[@]}"; do
docker run --rm -it \
-e X_DISTRO=$distro \
docker run --rm -it \
-e X_ARCH=$arch \
-u $(id -u):$(id -g) \
-w /workspace \
-v $temp_dir:/workspace \
sftpserver-builder:$distro \
sftpserver-builder \
/workspace/build.sh
done

ls -l .
cp sftp-server.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp scp.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp $temp_dir/sftp-server.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
cp $temp_dir/scp.*.bin $SCRIPT_DIR/../../src/ai/backend/runner
ls -lh src/ai/backend/runner

cd $SCRIPT_DIR/..
rm -rf "$temp_dir"
Loading

0 comments on commit 163c811

Please sign in to comment.