From 4731649ddca6331527cc1130ffe223c5e4e05654 Mon Sep 17 00:00:00 2001 From: iphydf Date: Wed, 11 Dec 2024 11:12:21 +0000 Subject: [PATCH] chore: Fix tsan build on circleci. --- .circleci/config.yml | 106 ++++++++++++---------------- .github/workflows/ci.yml | 60 ++++++---------- .github/workflows/docker.yml | 2 +- other/docker/circleci/Dockerfile | 8 +-- other/docker/circleci/entrypoint.sh | 4 +- other/docker/circleci/run | 8 ++- 6 files changed, 80 insertions(+), 108 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8692cd0a1b..4349e2f622 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,21 +3,16 @@ version: 2 workflows: version: 2 - program-analysis: - # TODO(iphydf): Re-enable tsan when it's fixed. + circleci: jobs: - # Dynamic analysis in the Bazel build - - bazel-asan - - bazel-msan - # - bazel-tsan - # Dynamic analysis with CMake - - asan - # - tsan - - ubsan - # Static analysis - - clang-analyze - - cpplint - - static-analysis + # - bazel-asan + # - bazel-msan + # - clang-analyze + # - cpplint + # - static-analysis + - mypy + - cimplefmt + - generate-events jobs: bazel-asan: @@ -30,20 +25,6 @@ jobs: - run: .circleci/bazel-test //c-toxcore/... - bazel-tsan: - working_directory: /tmp/cirrus-ci-build - docker: - - image: toxchat/toktok-stack:latest-tsan - - steps: - - checkout - - run: .circleci/bazel-test - //c-toxcore/... - -//c-toxcore/auto_tests:conference_av_test - -//c-toxcore/auto_tests:conference_test - -//c-toxcore/auto_tests:onion_test - -//c-toxcore/auto_tests:tox_many_test - bazel-msan: working_directory: /tmp/cirrus-ci-build docker: @@ -54,7 +35,7 @@ jobs: - run: .circleci/bazel-test //c-toxcore/auto_tests:lossless_packet_test - asan: + static-analysis: working_directory: ~/work docker: - image: ubuntu @@ -78,11 +59,19 @@ jobs: llvm-dev ninja-build pkg-config + - run: + apt-get install -y --no-install-recommends + ca-certificates + g++ + llvm-dev - checkout - run: git submodule update --init --recursive - - run: CC=clang .circleci/cmake-asan + - run: other/analysis/check_includes + - run: other/analysis/check_logger_levels + - run: other/analysis/run-clang + - run: other/analysis/run-gcc - tsan: + clang-analyze: working_directory: ~/work docker: - image: ubuntu @@ -91,59 +80,56 @@ jobs: - run: *apt_install - checkout - run: git submodule update --init --recursive - - run: CC=clang .circleci/cmake-tsan + - run: other/analysis/run-clang-analyze - ubsan: + cpplint: working_directory: ~/work docker: - image: ubuntu steps: - run: *apt_install + - run: + apt-get install -y --no-install-recommends + cpplint - checkout - run: git submodule update --init --recursive - - run: CC=clang .circleci/cmake-ubsan + - run: other/analysis/run-cpplint - static-analysis: + mypy: working_directory: ~/work - docker: - - image: ubuntu + docker: [{ image: alpine:latest }] steps: - - run: *apt_install - - run: - apt-get install -y --no-install-recommends - ca-certificates - g++ - llvm-dev + - run: apk add git openssh-client python3 py3-pip + - run: | + python3 -m venv /path/to/venv + . /path/to/venv/bin/activate + pip3 install mypy - checkout - run: git submodule update --init --recursive - - run: other/analysis/check_includes - - run: other/analysis/check_logger_levels - - run: other/analysis/run-clang - - run: other/analysis/run-gcc + - run: | + . /path/to/venv/bin/activate + (find . -name "*.py" -and -not -name "conanfile.py"; grep -lR '^#!.*python') \ + | xargs -n1 -P8 mypy --strict - clang-analyze: + cimplefmt: working_directory: ~/work - docker: - - image: ubuntu + docker: [{ image: alpine:latest }] steps: - - run: *apt_install + - run: apk add docker-cli-buildx git openssh-client - checkout + - setup_remote_docker - run: git submodule update --init --recursive - - run: other/analysis/run-clang-analyze + - run: other/docker/cimplefmt/run -u $(find tox* -name "*.[ch]") - cpplint: + generate-events: working_directory: ~/work - docker: - - image: ubuntu + machine: { image: ubuntu-2204:current } steps: - - run: *apt_install - - run: - apt-get install -y --no-install-recommends - cpplint - checkout - run: git submodule update --init --recursive - - run: other/analysis/run-cpplint + - run: other/event_tooling/run + - run: git diff --exit-code diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46aa57f315..859c0bab13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: ci on: - pull_request: + paull_request: branches: [master] # Cancel old PR builds when pushing new commits. @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: tool: [autotools, clang-tidy, compcert, cppcheck, doxygen, goblint, infer, freebsd, misra, modules, pkgsrc, rpm, slimcc, sparse, tcc, tokstyle] - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -34,37 +34,34 @@ jobs: with: file: other/docker/${{ matrix.tool }}/${{ matrix.tool }}.Dockerfile - coverage-linux: - runs-on: ubuntu-latest + sanitizer: + strategy: + fail-fast: false + matrix: + sanitizer: [asan, tsan, ubsan] + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 with: - submodules: recursive - - name: Build, test, and upload coverage - run: other/docker/coverage/run - - generate-events: - runs-on: ubuntu-latest - steps: + driver: docker - uses: actions/checkout@v4 with: submodules: recursive - - name: Run generate_event_c - run: | - other/event_tooling/run - git diff --exit-code + - name: Run sanitizer + run: other/docker/circleci/run "${{ matrix.sanitizer }}" - cimplefmt: - runs-on: ubuntu-latest + coverage-linux: + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Run cimplefmt - run: other/docker/cimplefmt/run -u $(find tox* -name "*.[ch]") + - name: Build, test, and upload coverage + run: other/docker/coverage/run build-android: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -111,7 +108,7 @@ jobs: ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 --build-config Debug build-netbsd: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -146,7 +143,7 @@ jobs: ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 build-freebsd: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -181,20 +178,3 @@ jobs: -DAUTOTEST=ON cmake --build . --target install ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 - - mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Python 3.9 - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Install mypy - run: pip install mypy - - name: Run mypy - run: | - (find . -name "*.py" -and -not -name "conanfile.py"; grep -lR '^#!.*python') \ - | xargs -n1 -P8 mypy --strict diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9b7092fc55..cf33e55623 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ name: docker on: push: branches: [master] - pull_request: + paull_request: branches: [master] # Cancel old PR builds when pushing new commits. diff --git a/other/docker/circleci/Dockerfile b/other/docker/circleci/Dockerfile index 3b23eed793..cb431c3d9f 100644 --- a/other/docker/circleci/Dockerfile +++ b/other/docker/circleci/Dockerfile @@ -1,18 +1,18 @@ ################################################ # cmake-asan -FROM ubuntu:20.04 +FROM ubuntu:24.04 RUN apt-get update && \ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ clang \ cmake \ + libclang-rt-dev \ libconfig-dev \ libgmock-dev \ libgtest-dev \ libopus-dev \ libsodium-dev \ libvpx-dev \ - llvm-dev \ ninja-build \ pkg-config \ && apt-get clean \ @@ -22,8 +22,8 @@ COPY entrypoint.sh / RUN ["chmod", "755", "/entrypoint.sh"] WORKDIR /home/builder -RUN groupadd -r -g 1000 builder \ - && useradd --no-log-init -r -g builder -u 1000 builder \ +RUN groupadd -r -g 987 builder \ + && useradd --no-log-init -r -g builder -u 987 builder \ && chown builder:builder /home/builder USER builder diff --git a/other/docker/circleci/entrypoint.sh b/other/docker/circleci/entrypoint.sh index 971759235f..0788e71d51 100755 --- a/other/docker/circleci/entrypoint.sh +++ b/other/docker/circleci/entrypoint.sh @@ -1,9 +1,9 @@ #!/bin/sh -set -eu +set -eux SANITIZER="${1:-asan}" cp -a /c-toxcore . cd c-toxcore -.circleci/cmake-"$SANITIZER" +.circleci/cmake-"$SANITIZER" || (cat /home/builder/c-toxcore/_build/CMakeFiles/CMakeError.log && false) diff --git a/other/docker/circleci/run b/other/docker/circleci/run index 668e0fc865..4e114176ed 100755 --- a/other/docker/circleci/run +++ b/other/docker/circleci/run @@ -1,6 +1,12 @@ #!/bin/sh +set -eux + SANITIZER="${1:-asan}" docker build -t toxchat/c-toxcore:circleci other/docker/circleci -docker run --name toxcore-circleci --rm -it -v "$PWD:/c-toxcore" toxchat/c-toxcore:circleci "$SANITIZER" +if [ -t 0 ]; then + docker run --name toxcore-circleci --rm --interactive --tty --volume "$PWD:/c-toxcore" toxchat/c-toxcore:circleci "$SANITIZER" +else + docker run --name toxcore-circleci --rm --volume "$PWD:/c-toxcore" toxchat/c-toxcore:circleci "$SANITIZER" +fi