feat: ci testing (#1)

* feat: ci for docker building * feat: distingu * chore: remove useless makefile * feat: add ten prefix * feat: build for multiplatform * fix: build error * feat: remove arm64 * feat: add ubuntu18 * feat: build linux * fix: syntax * fix: image * fix: tgn * feat: reduce * fix: downgrade to older checkout * fix: 2204 gcc * fix: dotenv and jinja2 * feat: downgrape node for ubuntu18 to avoid GLIBC min version requirements error * fix: cbindgen * feat: try v4 with node16 * feat: install latest git on ubuntu 18 * fix: node16 requires checkout v3 * fix: command not found * fix: cargo not found * fix: rust path * fix: explicity default nightly * feat: bring back g++-multilib * fix: cbindgen * feat: install cbindgen * fix: cbindgen * feat: options * feat: only build release and disable test * feat: add tree * fix: purge clang6 * feat: enable debug * feat: enable tests * feat: install clang18 * feat: disable test * feat: add build clang * feat: upload artifacts * feat: add devcontainer config * chore: ignore dev container * feat: unify name * feat: autoload env and submodule * fix: clang path of libc++.a * feat: upload clang * fix: set go path * fix: build clang release only * feat: enable debug with workaround * feat: remove clan * chore: ignore gen|out * chore: remove comment * fix: ubuntu 18 * test: main-ci * fix: downgrade to avoid glibc dependency
wangyoucao577 · Sep 15, 2024 · a38cea3 · a38cea3
1 parent ad5ae36
commit a38cea3
Show file tree

Hide file tree

Showing 7 changed files with 324 additions and 8 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,22 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
+{
+	"name": "ten_framework",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "ghcr.io/wangyoucao577/ten_building_ubuntu2204:feature-ci",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/git:1": {}
+	},
+	"remoteEnv": {
+		"PATH": "${containerWorkspaceFolder}/core/ten_gn:${containerEnv:PATH}"
+	},
+	"postCreateCommand": "git submodule update --init && export PATH=$(pwd)/core/ten_gn:$PATH"
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Configure tool-specific properties.
+	// "customizations": {},
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}
diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml
@@ -0,0 +1,36 @@
+name: Build - docker
+
+on:
+  push:
+    branches:
+    - '**'
+    paths:
+    - 'tools/docker_for_building/**'
+    - '.github/workflows/build_docker.yml'
+
+env:
+  IMAGE_NAME_PREFIX: ten_building_
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        include:
+          - image_name: "ubuntu2204"
+            workdir: "tools/docker_for_building/ubuntu/22.04"
+          - image_name: "ubuntu1804"
+            workdir: "tools/docker_for_building/ubuntu/18.04"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Build & Publish to Github Container Registry
+      uses: elgohr/Publish-Docker-Github-Action@v5
+      with:
+        name: ${{ github.repository_owner }}/${{ env.IMAGE_NAME_PREFIX }}${{ matrix.image_name }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+        registry: ghcr.io
+        workdir: ${{ matrix.workdir }}
+        snapshot: true
+        tags: "${{ github.ref == 'refs/heads/main-ci' && 'latest,' || '' }}"
diff --git a/.github/workflows/build_linux_ubuntu1804.yml b/.github/workflows/build_linux_ubuntu1804.yml
@@ -0,0 +1,52 @@
+name: Build - linux ubuntu1804
+on:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "**"
+    paths-ignore:
+      - "tools/**"
+      - "docs/**"
+      - ".vscode/**"
+      - ".devcontainer/**"
+      - ".github/configs/**"
+      - "**.md"
+  pull_request:
+
+jobs:
+  build-gcc:
+    runs-on: ubuntu-22.04
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+    strategy:
+      matrix:
+        build_type: [release]
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/ten_building_ubuntu1804:feature-ci
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+      - name: Build
+        run: |
+          export PATH=$(pwd)/core/ten_gn/:$PATH
+          echo $PATH
+          go env -w GOFLAGS="-buildvcs=false"
+          rustup default nightly
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_test=false ten_enable_package_manager=false
+          tgn build linux x64 ${{ matrix.build_type }}
+          tree -I 'gen|obj' out
+
+      - name: Upload ten_packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: ten_packages-ubuntu1804-gcc-${{ matrix.build_type }}
+          path: |
+            out/linux/x64/ten_packages/system
+            out/linux/x64/ten_packages/extension_group
+            out/linux/x64/ten_packages/extension/default_extension_cpp
+            out/linux/x64/ten_packages/extension/default_extension_go
+            out/linux/x64/ten_packages/extension/default_extension_python
+            out/linux/x64/ten_packages/extension/py_init_extension_cpp
diff --git a/.github/workflows/build_linux_ubuntu2204.yml b/.github/workflows/build_linux_ubuntu2204.yml
@@ -0,0 +1,95 @@
+name: Build - linux ubuntu2204
+on:
+  push:
+    branches:
+      - "**"
+    tags:
+      - "**"
+    paths-ignore:
+      - "tools/**"
+      - "docs/**"
+      - ".vscode/**"
+      - ".devcontainer/**"
+      - ".github/configs/**"
+      - "**.md"
+  pull_request:
+
+jobs:
+  build-gcc:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        build_type: [debug, release]
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/ten_building_ubuntu2204:feature-ci
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+      - name: Build
+        run: |
+          export PATH=$(pwd)/core/ten_gn:$PATH
+          echo $PATH
+          go env -w GOFLAGS="-buildvcs=false"
+          rustup default nightly
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=false log_level=1 enable_serialized_actions=true ten_enable_test=false
+          tgn build linux x64 ${{ matrix.build_type }}
+          tree -I 'gen|obj' out
+
+      - name: Upload tman
+        uses: actions/upload-artifact@v4
+        with:
+          name: tman-ubuntu2204-gcc-${{ matrix.build_type }}
+          path: out/linux/x64/ten_manager/bin/tman
+      - name: Upload ten_packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: ten_packages-ubuntu2204-gcc-${{ matrix.build_type }}
+          path: |
+            out/linux/x64/ten_packages/system
+            out/linux/x64/ten_packages/extension_group
+            out/linux/x64/ten_packages/extension/default_extension_cpp
+            out/linux/x64/ten_packages/extension/default_extension_go
+            out/linux/x64/ten_packages/extension/default_extension_python
+            out/linux/x64/ten_packages/extension/py_init_extension_cpp
+
+  build-clang:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        build_type: [debug, release]
+    container:
+      image: ghcr.io/${{ github.repository_owner }}/ten_building_ubuntu2204:feature-ci
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0"
+          submodules: "true"
+      - name: Build
+        run: |
+          export PATH=$(pwd)/core/ten_gn:$PATH
+          echo $PATH
+          go env -w GOFLAGS="-buildvcs=false"
+          go1.20.12 download
+          rustup default nightly
+          tgn gen linux x64 ${{ matrix.build_type }} -- is_clang=true log_level=1 enable_serialized_actions=true ten_enable_test=false
+          tgn build linux x64 ${{ matrix.build_type }}
+          tree -I 'gen|obj' out
+
+      - name: Upload tman
+        uses: actions/upload-artifact@v4
+        with:
+          name: tman-ubuntu2204-clang-${{ matrix.build_type }}
+          path: out/linux/x64/ten_manager/bin/tman
+      - name: Upload ten_packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: ten_packages-ubuntu2204-clang-${{ matrix.build_type }}
+          path: |
+            out/linux/x64/ten_packages/system
+            out/linux/x64/ten_packages/extension_group
+            out/linux/x64/ten_packages/extension/default_extension_cpp
+            out/linux/x64/ten_packages/extension/default_extension_go
+            out/linux/x64/ten_packages/extension/default_extension_python
+            out/linux/x64/ten_packages/extension/py_init_extension_cpp
diff --git a/tools/docker_for_building/ubuntu/18.04/Dockerfile b/tools/docker_for_building/ubuntu/18.04/Dockerfile
@@ -0,0 +1,112 @@
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get clean && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends apt-utils software-properties-common && \
+  add-apt-repository ppa:git-core/ppa && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends curl \
+  lsb-release \
+  iwyu \
+  libssl-dev \
+  p7zip-full \
+  p7zip-rar \
+  git \
+  tree \
+  libasan5 \
+  autoconf \
+  libtool \
+  uuid-dev \
+  libmsgpack-dev \
+  libmysqlclient-dev \
+  libmysqlcppconn-dev \
+  libcrypto++-dev \
+  libexpat1-dev \
+  libxdamage1 \
+  make \
+  vim \
+  libxcomposite-dev \
+  libdrm-dev \
+  libavformat-dev \
+  libavfilter-dev \
+  libx264-dev \
+  build-essential \
+  zlib1g-dev \
+  libncurses5-dev \
+  libgdbm-dev \
+  libnss3-dev \
+  libssl-dev \
+  libreadline-dev \
+  libffi-dev \
+  wget \
+  gpg-agent
+
+# ================
+# Installing cmake
+#
+# The default version of cmake of Ubuntu 18.04 is 3.10, but TEN needs cmake version greater than 3.14, so we need to install it from cmake official site.
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \
+  && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends cmake
+
+# ===============
+# Installing gcc13
+#
+# NOTE: install gcc after cmake, while cmake will auto-install gcc7.
+# Install gcc13 as default.
+RUN apt-get purge -y gcc gcc-7 g++-7 && \
+  apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
+  apt-get update && \
+  apt-get install -y --no-install-recommends gcc-13 g++-13 && \
+  apt-get install -y gcc-9-multilib g++-13-multilib && \
+  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \
+  update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100
+
+# ==============
+# Install golang
+
+# /root/go is GOPATH.
+ENV PATH="$PATH:/usr/local/go/bin:/root/go/bin"
+
+# TEN go binding needs to be compatible with GO 1.20, so we need to install GO
+# 1.20 to check the compatibility.
+RUN curl -OL https://go.dev/dl/go1.22.3.linux-amd64.tar.gz && \
+  rm -rf /usr/local/go && tar -C /usr/local -xvf go1.22.3.linux-amd64.tar.gz && rm go1.22.3.linux-amd64.tar.gz && \
+  go install golang.org/dl/go1.20.12@latest && go1.20.12 download
+
+# =======================
+# Install cargo for rust.
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
+  /root/.cargo/bin/rustup install nightly && \
+  /root/.cargo/bin/rustup default nightly && \
+  /root/.cargo/bin/cargo install --force cbindgen
+
+ENV PATH="/root/.cargo/bin:$PATH"
+
+# =======================================
+# Installing Python3.10 from source code.
+ENV PYTHON_VERSION=3.10.14
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
+  tar -zxvf Python-${PYTHON_VERSION}.tgz && \
+  cd Python-${PYTHON_VERSION} && \
+  ./configure --with-lto --with-computed-gotos --with-system-ffi --prefix=/usr --disable-test-modules --enable-shared && \
+  make -j && \
+  make altinstall && \
+  rm -rf Python-{PYTHON_VERSION}*
+
+RUN ln -sf /usr/bin/python3.10-config /usr/bin/python3-config && \
+  ln -sf /usr/bin/python3.10 /usr/bin/python3 && \
+  ln -sf /usr/bin/python3.10 /usr/bin/python && \
+  ln -sf /usr/bin/pip3.10 /usr/bin/pip3 && \
+  cp /usr/lib/python3/dist-packages/lsb_release.py /usr/bin && \
+  pip3 install --upgrade setuptools && \
+  pip3 install --upgrade pip && \
+  pip3 install requests oss2 && \
+  pip3 install --use-pep517 python-dotenv jinja2
+
+# Cleanup
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
diff --git a/tools/docker_for_building/ubuntu/22.04/Dockerfile b/tools/docker_for_building/ubuntu/22.04/Dockerfile
@@ -19,6 +19,7 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend
   p7zip-full \
   p7zip-rar \
   git \
+  tree \
   libasan5 \
   autoconf \
   libtool \
@@ -56,6 +57,7 @@ RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommend
 
 RUN pip3 install --upgrade setuptools pip && \
   pip3 install requests oss2 && \
+  pip3 install --use-pep517 python-dotenv jinja2 && \
   ln -sf /usr/bin/python3.10-config /usr/bin/python3-config && \
   ln -sf /usr/bin/python3 /usr/bin/python
 
@@ -81,7 +83,8 @@ RUN wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key
   update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-18 100 && \
   update-alternatives --install /usr/bin/clang-check clang-check /usr/bin/clang-check-18 100 && \
   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-18 100 && \
-  ln -sf /usr/lib/llvm-18/lib/x86_64-pc-linux-gnu/libc++.a /usr/lib/x86_64-linux-gnu/libc++.a && \
+  mkdir -p /usr/lib/llvm-18/lib/x86_64-pc-linux-gnu && \
+  ln -sf $(clang -print-file-name=libc++.a) /usr/lib/llvm-18/lib/x86_64-pc-linux-gnu && \
   ln -sf $(clang -print-file-name=libclang_rt.asan.so) $(dirname $(clang -print-file-name=libclang_rt.asan.so))/libclang_rt.asan-x86_64.so
 
 # ==============
@@ -100,9 +103,10 @@ RUN curl -OL https://go.dev/dl/go1.22.3.linux-amd64.tar.gz && \
 # Install cargo for rust.
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
   /root/.cargo/bin/rustup install nightly && \
-  /root/.cargo/bin/rustup default nightly
+  /root/.cargo/bin/rustup default nightly && \
+  /root/.cargo/bin/cargo install --force cbindgen
 
-ENV PATH="$HOME/.cargo/bin:$PATH"
+ENV PATH="/root/.cargo/bin:$PATH"
 
 # Cleanup
 RUN apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*
diff --git a/tools/docker_for_building/ubuntu/22.04/Makefile b/tools/docker_for_building/ubuntu/22.04/Makefile