diff --git a/.cicd/package-builder.sh b/.cicd/package-builder.sh deleted file mode 100755 index dd26885e208..00000000000 --- a/.cicd/package-builder.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -if [[ $(uname) == 'Darwin' ]]; then - echo 'Darwin family detected, building for brew.' - [[ -z $ARTIFACT ]] && ARTIFACT='*.rb;*.tar.gz' - PACKAGE_TYPE='brew' -else - . /etc/os-release - if [[ "$ID_LIKE" == 'debian' || "$ID" == 'debian' ]]; then - echo 'Debian family detected, building for dpkg.' - [[ -z $ARTIFACT ]] && ARTIFACT='*.deb' - PACKAGE_TYPE='deb' - elif [[ "$ID_LIKE" == 'rhel fedora' || "$ID" == 'fedora' ]]; then - echo 'Fedora family detected, building for RPM.' - [[ -z $ARTIFACT ]] && ARTIFACT='*.rpm' - PACKAGE_TYPE='rpm' - mkdir -p ~/rpmbuild/BUILD - mkdir -p ~/rpmbuild/BUILDROOT - mkdir -p ~/rpmbuild/RPMS - mkdir -p ~/rpmbuild/SOURCES - mkdir -p ~/rpmbuild/SPECS - mkdir -p ~/rpmbuild/SRPMS - yum install -y rpm-build - elif [[ $ID == 'amzn' ]]; then - echo "SKIPPED: We do not generate $NAME packages since they use rpms created from Centos." - exit 0 - else - echo 'ERROR: Could not determine which operating system this script is running on!' - uname - echo "ID_LIKE=\"$ID_LIKE\"" - cat /etc/os-release - exit 1 - fi -fi -BASE_COMMIT=$(cat build/programs/nodeos/config.hpp | grep 'version' | awk '{print $5}' | tr -d ';') -BASE_COMMIT="${BASE_COMMIT:2:42}" -echo "Found build against $BASE_COMMIT." -cd build/packages -chmod 755 ./*.sh -./generate_package.sh $PACKAGE_TYPE \ No newline at end of file diff --git a/.cicd/package.sh b/.cicd/package.sh index 5cc836fdb85..f1b4e6d23c6 100755 --- a/.cicd/package.sh +++ b/.cicd/package.sh @@ -4,11 +4,9 @@ set -eo pipefail mkdir -p $BUILD_DIR -PRE_COMMANDS="cd $MOUNTED_DIR/build/packages && chmod 755 ./*.sh" - if [[ $(uname) == 'Darwin' ]]; then - bash -c "$PRE_COMMANDS && ./generate_package.sh brew" + bash -c "cd build/packages && chmod 755 ./*.sh && ./generate_package.sh brew" ARTIFACT='*.rb;*.tar.gz' cd build/packages @@ -29,6 +27,7 @@ else # Linux . $HELPERS_DIR/docker-hash.sh + PRE_COMMANDS="cd $MOUNTED_DIR/build/packages && chmod 755 ./*.sh" if [[ "$IMAGE_TAG" =~ "ubuntu" ]]; then ARTIFACT='*.deb' diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml index 33a6d94bd87..41b6444fb49 100644 --- a/.cicd/pipeline.yml +++ b/.cicd/pipeline.yml @@ -70,177 +70,177 @@ steps: - "queue=mac-anka-large-node-fleet" skip: $SKIP_MOJAVE - # - wait + - wait - # - label: ":aws: Amazon_Linux 2 - Unit Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':aws: Amazon_Linux 2 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/parallel-tests.sh" - # env: - # IMAGE_TAG: "amazon_linux-2" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_AMAZON_LINUX_2}${SKIP_UNIT_TESTS} + - label: ":aws: Amazon_Linux 2 - Unit Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':aws: Amazon_Linux 2 - Build' && tar -xzf build.tar.gz" + - "./.cicd/parallel-tests.sh" + env: + IMAGE_TAG: "amazon_linux-2" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_UNIT_TESTS} - # - label: ":centos: CentOS 7.6 - Unit Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/parallel-tests.sh" - # env: - # IMAGE_TAG: "centos-7.6" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_CENTOS_7}${SKIP_UNIT_TESTS} + - label: ":centos: CentOS 7.6 - Unit Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz" + - "./.cicd/parallel-tests.sh" + env: + IMAGE_TAG: "centos-7.6" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7}${SKIP_UNIT_TESTS} - # - label: ":ubuntu: Ubuntu 16.04 - Unit Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/parallel-tests.sh" - # env: - # IMAGE_TAG: "ubuntu-16.04" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_UBUNTU_16}${SKIP_UNIT_TESTS} + - label: ":ubuntu: Ubuntu 16.04 - Unit Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz" + - "./.cicd/parallel-tests.sh" + env: + IMAGE_TAG: "ubuntu-16.04" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_16}${SKIP_UNIT_TESTS} - # - label: ":ubuntu: Ubuntu 18.04 - Unit Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/parallel-tests.sh" - # env: - # IMAGE_TAG: "ubuntu-18.04" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_UBUNTU_18}${SKIP_UNIT_TESTS} + - label: ":ubuntu: Ubuntu 18.04 - Unit Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" + - "./.cicd/parallel-tests.sh" + env: + IMAGE_TAG: "ubuntu-18.04" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_18}${SKIP_UNIT_TESTS} - # - label: ":darwin: macOS 10.14 - Unit Tests" - # command: - # - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl boost@1.70 cmake mongodb" - # - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive" - # - "cd eos && buildkite-agent artifact download build.tar.gz . --step ':darwin: macOS 10.14 - Build' && tar -xzf build.tar.gz" - # - "cd eos && ./.cicd/parallel-tests.sh" - # plugins: - # - chef/anka#v0.5.1: - # no-volume: true - # inherit-environment-vars: true - # vm-name: 10.14.4_6C_14G_40G - # vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - # always-pull: true - # debug: true - # wait-network: true - # agents: - # - "queue=mac-anka-node-fleet" - # skip: ${SKIP_MOJAVE}${SKIP_UNIT_TESTS} + - label: ":darwin: macOS 10.14 - Unit Tests" + command: + - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl boost@1.70 cmake mongodb" + - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive" + - "cd eos && buildkite-agent artifact download build.tar.gz . --step ':darwin: macOS 10.14 - Build' && tar -xzf build.tar.gz" + - "cd eos && ./.cicd/parallel-tests.sh" + plugins: + - chef/anka#v0.5.1: + no-volume: true + inherit-environment-vars: true + vm-name: 10.14.4_6C_14G_40G + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + always-pull: true + debug: true + wait-network: true + agents: + - "queue=mac-anka-node-fleet" + skip: ${SKIP_MOJAVE}${SKIP_UNIT_TESTS} - # - label: ":pipeline: Generate Serial Test Steps" - # command: "./.cicd/generate-serial-tests.sh | buildkite-agent pipeline upload" - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: $SKIP_GENERATE_SERIAL_TESTS + - label: ":pipeline: Generate Serial Test Steps" + command: "./.cicd/generate-serial-tests.sh | buildkite-agent pipeline upload" + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: $SKIP_GENERATE_SERIAL_TESTS - # - label: ":aws: Amazon_Linux 2 - Long-Running Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':aws: Amazon_Linux 2 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/long-running-tests.sh" - # env: - # IMAGE_TAG: "amazon_linux-2" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LONG_RUNNING_TESTS:-true} + - label: ":aws: Amazon_Linux 2 - Long-Running Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':aws: Amazon_Linux 2 - Build' && tar -xzf build.tar.gz" + - "./.cicd/long-running-tests.sh" + env: + IMAGE_TAG: "amazon_linux-2" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LONG_RUNNING_TESTS:-true} - # - label: ":centos: CentOS 7.6 - Long-Running Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/long-running-tests.sh" - # env: - # IMAGE_TAG: "centos-7.6" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_CENTOS_7}${SKIP_LONG_RUNNING_TESTS:-true} + - label: ":centos: CentOS 7.6 - Long-Running Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.6 - Build' && tar -xzf build.tar.gz" + - "./.cicd/long-running-tests.sh" + env: + IMAGE_TAG: "centos-7.6" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7}${SKIP_LONG_RUNNING_TESTS:-true} - # - label: ":ubuntu: Ubuntu 16.04 - Long-Running Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/long-running-tests.sh" - # env: - # IMAGE_TAG: "ubuntu-16.04" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_UBUNTU_16}${SKIP_LONG_RUNNING_TESTS:-true} + - label: ":ubuntu: Ubuntu 16.04 - Long-Running Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 16.04 - Build' && tar -xzf build.tar.gz" + - "./.cicd/long-running-tests.sh" + env: + IMAGE_TAG: "ubuntu-16.04" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_16}${SKIP_LONG_RUNNING_TESTS:-true} - # - label: ":ubuntu: Ubuntu 18.04 - Long-Running Tests" - # command: - # - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" - # - "./.cicd/long-running-tests.sh" - # env: - # IMAGE_TAG: "ubuntu-18.04" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_UBUNTU_18}${SKIP_LONG_RUNNING_TESTS:-true} + - label: ":ubuntu: Ubuntu 18.04 - Long-Running Tests" + command: + - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" + - "./.cicd/long-running-tests.sh" + env: + IMAGE_TAG: "ubuntu-18.04" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_18}${SKIP_LONG_RUNNING_TESTS:-true} - # - label: ":darwin: macOS 10.14 - Long-Running Tests" - # command: - # - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl boost@1.70 cmake mongodb ccache" - # - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive" - # - "cd eos && buildkite-agent artifact download build.tar.gz . --step ':darwin: macOS 10.14 - Build' && tar -xzf build.tar.gz" - # - "cd eos && ./.cicd/long-running-tests.sh" - # plugins: - # - chef/anka#v0.5.1: - # no-volume: true - # inherit-environment-vars: true - # vm-name: 10.14.4_6C_14G_40G - # vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - # modify-cpu: 12 - # modify-ram: 24 - # always-pull: true - # debug: true - # wait-network: true - # agents: - # - "queue=mac-anka-large-node-fleet" - # skip: ${SKIP_MOJAVE}${SKIP_LONG_RUNNING_TESTS:-true} + - label: ":darwin: macOS 10.14 - Long-Running Tests" + command: + - "brew install git graphviz libtool gmp llvm@4 pkgconfig python python@2 doxygen libusb openssl boost@1.70 cmake mongodb ccache" + - "git clone $BUILDKITE_REPO eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive" + - "cd eos && buildkite-agent artifact download build.tar.gz . --step ':darwin: macOS 10.14 - Build' && tar -xzf build.tar.gz" + - "cd eos && ./.cicd/long-running-tests.sh" + plugins: + - chef/anka#v0.5.1: + no-volume: true + inherit-environment-vars: true + vm-name: 10.14.4_6C_14G_40G + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + modify-cpu: 12 + modify-ram: 24 + always-pull: true + debug: true + wait-network: true + agents: + - "queue=mac-anka-large-node-fleet" + skip: ${SKIP_MOJAVE}${SKIP_LONG_RUNNING_TESTS:-true} - # - wait: - # continue_on_failure: true + - wait: + continue_on_failure: true - # - label: ":bar_chart: Test Metrics" - # command: | - # echo '+++ :compression: Extracting Test Metrics Code' - # tar -zxf .cicd/metrics/test-metrics.tar.gz - # echo '+++ :javascript: Running test-metrics.js' - # node --max-old-space-size=32768 test-metrics.js - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: 10 - # soft_fail: true + - label: ":bar_chart: Test Metrics" + command: | + echo '+++ :compression: Extracting Test Metrics Code' + tar -zxf .cicd/metrics/test-metrics.tar.gz + echo '+++ :javascript: Running test-metrics.js' + node --max-old-space-size=32768 test-metrics.js + agents: + queue: "automation-eos-builder-fleet" + timeout: 10 + soft_fail: true - wait - # - label: ":ubuntu: Ubuntu 18.04 - Contract Builder" - # command: - # - "./.cicd/installation-build.sh" - # env: - # IMAGE_TAG: "ubuntu-18.04" - # BUILDKITE_AGENT_ACCESS_TOKEN: - # agents: - # queue: "automation-eos-builder-fleet" - # timeout: ${TIMEOUT:-10} - # skip: ${SKIP_UBUNTU_18}${SKIP_CONTRACT_BUILDER} + - label: ":ubuntu: Ubuntu 18.04 - Contract Builder" + command: + - "./.cicd/installation-build.sh" + env: + IMAGE_TAG: "ubuntu-18.04" + BUILDKITE_AGENT_ACCESS_TOKEN: + agents: + queue: "automation-eos-builder-fleet" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_18}${SKIP_CONTRACT_BUILDER} - label: ":centos: CentOS 7.6 - Package Builder" command: