From a6a12b56fd28ae5bc4b72ec87ffcf7f68f3df59b Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Mon, 17 Jul 2023 16:41:57 -0700 Subject: [PATCH 1/4] Don't analyze __bro_plugin__ file content in tests, only its presence The plugin and plugin_tarfile tests baselined the contents of the __bro_plugin__ file, a Zeek-internal concern. zkg only cares about the presence/naming of that file, so the tests now focus on that. --- .../baselines/tests.plugin/__bro_plugin__.after_enabling | 2 -- .../tests.plugin/plugins.packages.rot13.__bro_plugin__ | 2 -- .../plugins.packages.rot13.__bro_plugin__.disabled | 2 -- .../plugins.packages.rot13.__bro_plugin__ | 2 -- testing/tests/plugin | 7 +++---- testing/tests/plugin_tarfile | 2 +- 6 files changed, 4 insertions(+), 13 deletions(-) delete mode 100644 testing/baselines/tests.plugin/__bro_plugin__.after_enabling delete mode 100644 testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__ delete mode 100644 testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__.disabled delete mode 100644 testing/baselines/tests.plugin_tarfile/plugins.packages.rot13.__bro_plugin__ diff --git a/testing/baselines/tests.plugin/__bro_plugin__.after_enabling b/testing/baselines/tests.plugin/__bro_plugin__.after_enabling deleted file mode 100644 index ae94f48d..00000000 --- a/testing/baselines/tests.plugin/__bro_plugin__.after_enabling +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Demo::Rot13 diff --git a/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__ b/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__ deleted file mode 100644 index ae94f48d..00000000 --- a/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__ +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Demo::Rot13 diff --git a/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__.disabled b/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__.disabled deleted file mode 100644 index ae94f48d..00000000 --- a/testing/baselines/tests.plugin/plugins.packages.rot13.__bro_plugin__.disabled +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Demo::Rot13 diff --git a/testing/baselines/tests.plugin_tarfile/plugins.packages.rot13.__bro_plugin__ b/testing/baselines/tests.plugin_tarfile/plugins.packages.rot13.__bro_plugin__ deleted file mode 100644 index ae94f48d..00000000 --- a/testing/baselines/tests.plugin_tarfile/plugins.packages.rot13.__bro_plugin__ +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -Demo::Rot13 diff --git a/testing/tests/plugin b/testing/tests/plugin index 2c6eef6f..efc18c6c 100644 --- a/testing/tests/plugin +++ b/testing/tests/plugin @@ -3,7 +3,7 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC: zkg install rot13 -# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ +# @TEST-EXEC: test -f plugins/packages/rot13/__bro_plugin__ # @TEST-EXEC: btest-diff scripts/packages/rot13/__load__.zeek # Unloading the package should also disable the plugin, which we @@ -11,14 +11,13 @@ # @TEST-EXEC: zkg unload rot13 # @TEST-EXEC: test ! -f plugins/packages/rot13/__bro_plugin__ -# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__.disabled +# @TEST-EXEC: test -f plugins/packages/rot13/__bro_plugin__.disabled # (Re-)loading the package should also (re-)enable the plugin. # @TEST-EXEC: zkg load rot13 +# @TEST-EXEC: test -f plugins/packages/rot13/__bro_plugin__ # @TEST-EXEC: test ! -f plugins/packages/rot13/__bro_plugin__.disabled -# @TEST-EXEC: cp plugins/packages/rot13/__bro_plugin__ __bro_plugin__.after_enabling -# @TEST-EXEC: btest-diff __bro_plugin__.after_enabling echo "$(pwd)/packages/rot13" >> sources/one/bob/zkg.index cd sources/one diff --git a/testing/tests/plugin_tarfile b/testing/tests/plugin_tarfile index d0c64f78..f3840406 100644 --- a/testing/tests/plugin_tarfile +++ b/testing/tests/plugin_tarfile @@ -3,7 +3,7 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC: zkg install rot13 -# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ +# @TEST-EXEC: test -f plugins/packages/rot13/__bro_plugin__ # @TEST-EXEC: btest-diff scripts/packages/rot13/__load__.zeek cd packages/rot13 From ee7845e6bd0f068714c1239a2df8ae8e25d99cd1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 30 Jun 2023 18:52:49 -0700 Subject: [PATCH 2/4] CI: remove Cirrus setup The Cirrus setup uses Docker containers to install our binary packages, which these days is not required -- just use our own Docker images. --- .cirrus.yml | 41 ------------------------ .github/workflows/ci-notification.yml | 22 ------------- ci/test.sh | 7 ----- ci/zeek/Dockerfile | 45 --------------------------- 4 files changed, 115 deletions(-) delete mode 100644 .cirrus.yml delete mode 100644 .github/workflows/ci-notification.yml delete mode 100755 ci/test.sh delete mode 100644 ci/zeek/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index f034e260..00000000 --- a/.cirrus.yml +++ /dev/null @@ -1,41 +0,0 @@ -cpus: &CPUS 2 - -resources_template: &RESOURCES_TEMPLATE - cpu: *CPUS - memory: 4GB - -env: - ZEEK_CI_CPUS: *CPUS - -ci_template: &CI_TEMPLATE - only_if: > - $CIRRUS_PR != '' || - $CIRRUS_BRANCH == 'master' || - $CIRRUS_BRANCH =~ 'release/.*' - - test_script: ci/test.sh - - on_failure: - upload_btest_tmp_dir_artifacts: - path: "testing/**/tmp.tar.gz" - always: - upload_btest_xml_results_artifacts: - path: "testing/**/btest-results.xml" - type: text/xml - format: junit - -zeek_lts_task: - container: - dockerfile: ci/zeek/Dockerfile - docker_arguments: - zeek_package: zeek-lts - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - -zeek_feature_task: - container: - dockerfile: ci/zeek/Dockerfile - docker_arguments: - zeek_package: zeek - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE diff --git a/.github/workflows/ci-notification.yml b/.github/workflows/ci-notification.yml deleted file mode 100644 index 6ddc0fc6..00000000 --- a/.github/workflows/ci-notification.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: CI Email Notification -on: - check_suite: - types: [completed] -jobs: - notify: - if: github.repository == 'zeek/package-manager' - runs-on: ubuntu-latest - steps: - - name: Send CI Email Notification - uses: zeek/ci-email-action@master - env: - CI_APP_NAME: "Cirrus CI" - BRANCH_WHITELIST: "master|release/.*" - SMTP_HOST: ${{ secrets.SMTP_HOST }} - SMTP_PORT: ${{ secrets.SMTP_PORT }} - SMTP_USER: ${{ secrets.SMTP_USER }} - SMTP_PASS: ${{ secrets.SMTP_PASS }} - MAIL_FROM: ${{ secrets.MAIL_FROM }} - MAIL_TO: ${{ secrets.MAIL_TO }} - MAIL_REPLY_TO: ${{ secrets.MAIL_REPLY_TO }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/test.sh b/ci/test.sh deleted file mode 100755 index 8f2c6f98..00000000 --- a/ci/test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash - -result=0 -cd testing -btest -d -b -x btest-results.xml -j ${ZEEK_CI_CPUS} || result=1 -[[ -d .tmp ]] && tar -czf tmp.tar.gz .tmp -exit ${result} diff --git a/ci/zeek/Dockerfile b/ci/zeek/Dockerfile deleted file mode 100644 index fdbce186..00000000 --- a/ci/zeek/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM centos:7 - -ARG zeek_package=zeek - -RUN yum -y update ca-certificates \ - && yum clean all && rm -rf /var/cache/yum - -RUN yum -y install \ - epel-release \ - && yum clean all && rm -rf /var/cache/yum - -RUN yum -y install \ - centos-release-scl \ - && yum clean all && rm -rf /var/cache/yum - -RUN yum -y install \ - devtoolset-7 \ - && yum clean all && rm -rf /var/cache/yum - -RUN yum -y install \ - cmake \ - cmake3 \ - git \ - python3 \ - python3-pip\ - wget \ - && yum clean all && rm -rf /var/cache/yum - -RUN cd /etc/yum.repos.d/ \ - && wget https://download.opensuse.org/repositories/security:zeek/CentOS_7/security:zeek.repo \ - && yum -y install $zeek_package \ - && yum clean all && rm -rf /var/cache/yum - -RUN pip3 install GitPython semantic_version - -RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ - echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env - -ENV BASH_ENV="/usr/bin/zeek-ci-env" \ - ENV="/usr/bin/zeek-ci-env" \ - PROMPT_COMMAND=". /usr/bin/zeek-ci-env" \ - PATH="${PATH}:/opt/zeek/bin" - -RUN git config --global user.email "zeek@example.com" && \ - git config --global user.name "Zeek" From 3cdfeb17fc057109a38aefcc8bb8d154fdd78c45 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 30 Jun 2023 18:53:57 -0700 Subject: [PATCH 3/4] CI: add Github workflow for testing zkg --- .github/workflows/test.yml | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..05060fef --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: Test and upload Python package + +on: + pull_request: + push: + branches: [master] + tags: + - 'v*' + - '!v*-dev' + +env: + ZEEKROOT: /usr/local/zeek + +jobs: + test: + strategy: + fail-fast: false + matrix: + # Test Zeek LTS, latest, and nightly, and + # don't fail for the nightly one. + include: + - repo: zeek + version: latest + continue_on_error: false + - repo: zeek + version: lts + continue_on_error: false + - repo: zeek-dev + version: latest + continue_on_error: true + + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.continue_on_error }} + container: + image: zeek/${{ matrix.repo }}:${{ matrix.version }} + + steps: + - name: Install build environment + run: | + apt-get update + apt-get install -y --no-install-recommends cmake g++ libssl-dev libpcap-dev make + - name: Remove zkg installation + # Rule out confusion between test environment and pre-existing zkg: + run: | + rm $ZEEKROOT/bin/zkg $ZEEKROOT/share/man/man1/zkg.1 + rm -r $ZEEKROOT/etc/zkg + rm -r $ZEEKROOT/lib/zeek/python/zeekpkg + rm -r $ZEEKROOT/var/lib/zkg + - uses: actions/checkout@v3 + - name: Run unit tests + run: btest -j -A -d -c testing/btest.cfg + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: btest-${{ matrix.repo }}-${{ matrix.version }} + path: testing/.tmp/ From 0db0afaa7da31e2e236fb4e9c888fcdceb801ab1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Sun, 2 Jul 2023 22:20:46 -0700 Subject: [PATCH 4/4] CI: automatically push to PyPI when pushing a git tag --- .github/workflows/test.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 05060fef..2519aee8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,3 +54,24 @@ jobs: with: name: btest-${{ matrix.repo }}-${{ matrix.version }} path: testing/.tmp/ + + upload: + runs-on: ubuntu-latest + needs: [test] + if: github.repository == 'zeek/package-manager' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/checkout@v3 + - name: Check release version + # This fails e.g. if VERSION contains a dev commits suffix, + # since we don't want to push these to PyPI. Accepts two- + # and three-component version numbers (e.g. 1.0 and 1.0.1). + run: | + grep -E -x '[0-9]+\.[0-9]+(\.[0-9]+)?' VERSION + - name: Build wheel + run: | + make dist + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }}