Build script: #236
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push, pull_request, workflow_dispatch] | |
| jobs: | |
| rat: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: rat | |
| run: make rat clean | |
| linux-build-docker: | |
| strategy: | |
| matrix: | |
| distro: [ubuntu, centos, debian, i386/ubuntu] | |
| exclude: | |
| - distro: ubuntu | |
| - distro: centos | |
| - distro: debian | |
| - distro: i386/ubuntu | |
| include: | |
| - distro: ubuntu | |
| ver: 22.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 20.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 18.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 18.04 | |
| codecov: yes | |
| btype: release | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 18.04 | |
| codecov: no | |
| btype: debug | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 18.04 | |
| codecov: no | |
| btype: release | |
| buser: root | |
| - distro: ubuntu | |
| ver: 16.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: ubuntu | |
| ver: 14.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: debian | |
| ver: 10 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: centos | |
| ver: 6 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: centos | |
| ver: 7 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: centos | |
| ver: 8 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| - distro: i386/ubuntu | |
| ver: 18.04 | |
| codecov: no | |
| btype: release | |
| buser: qfsbuild | |
| runs-on: ubuntu-latest | |
| env: | |
| BUILD_OS_NAME: linux | |
| BUILD_RUN_DOCKER: "yes" | |
| DISTRO: ${{ matrix.distro }} | |
| VER: ${{ matrix.ver }} | |
| CODECOV: ${{ matrix.codecov }} | |
| BTYPE: ${{ matrix.btype }} | |
| BUSER: ${{ matrix.buser }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Linux docker build | |
| run: | | |
| travis/script.sh | |
| sudo chown -R "$USER" .git || true | |
| - name: Upload tarball | |
| if: | | |
| ('refs/heads/master' == github.ref | |
| || startsWith(github.ref, 'refs/tags/')) | |
| && 'release' == matrix.btype | |
| && 'qfsbuild' == matrix.buser | |
| && 'yes' != matrix.codecov | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-west-2 | |
| run: aws s3 cp build/qfs-*.tgz s3://quantcast-qfs | |
| osx-build: | |
| runs-on: macos-latest | |
| env: | |
| BUILD_OS_NAME: osx | |
| steps: | |
| - name: Brew install | |
| run: | | |
| brew install boost || true | |
| brew install macfuse || true | |
| brew install go || true | |
| brew install python3 || true | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: MacOS build | |
| run: travis/script.sh | |
| - name: Upload tarball | |
| if: | | |
| 'refs/heads/master' == github.ref | |
| || startsWith(github.ref, 'refs/tags/') | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-west-2 | |
| run: aws s3 cp build/qfs-*.tgz s3://quantcast-qfs | |
| osx-build-arm64: | |
| runs-on: macos-latest-xlarge | |
| env: | |
| BUILD_OS_NAME: osx | |
| steps: | |
| - name: Brew install | |
| run: | | |
| brew install boost || true | |
| brew install macfuse || true | |
| brew install go || true | |
| brew install awscli || true | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: MacOS build | |
| run: travis/script.sh | |
| - name: Upload tarball | |
| if: | | |
| 'refs/heads/master' == github.ref | |
| || startsWith(github.ref, 'refs/tags/') | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-west-2 | |
| run: aws s3 cp build/qfs-*.tgz s3://quantcast-qfs | |
| cygwin-build: | |
| if: ${{ false }} # turn off for now | |
| runs-on: windows-latest | |
| env: | |
| CYG_MIRROR: http://cygwin.mirror.constant.com | |
| DEPENDENCIES: gcc-g++,autoconf,automake,make,cmake,libboost-devel,libssl-devel,bzip2,python,git,libstdc++6-devel,libkrb5-devel,zlib-devel | |
| steps: | |
| - name: Set git to use LF | |
| run: | | |
| git config --global core.autocrlf false | |
| git config --global core.eol lf | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Hadoop tools, Cygwin, and do build | |
| shell: pwsh | |
| run: | | |
| $cur = (Get-Item -Path "." -Verbose).FullName | |
| $tools = "$cur\wtools" | |
| if (!(Test-Path $tools)) { | |
| New-Item -ItemType Directory -Force -Path $tools | Out-Null | |
| } | |
| $hadoopVer = "2.6.0" | |
| $hadoopPath = "$tools\hadoop" | |
| if (!(Test-Path $hadoopPath)) { | |
| New-Item -ItemType Directory -Force -Path $hadoopPath | Out-Null | |
| } | |
| Push-Location $hadoopPath | |
| Invoke-WebRequest ` | |
| -Uri "https://github.com/steveloughran/winutils/archive/master.zip" ` | |
| -OutFile "winutils-master.zip" | |
| # extract | |
| Invoke-Expression "7z.exe x winutils-master.zip" | |
| # add hadoop bin to environment variables | |
| $env:HADOOP_HOME = "$hadoopPath/winutils-master/hadoop-$hadoopVer" | |
| Pop-Location | |
| # Download and install Cygwin | |
| Push-Location $tools | |
| $cygwinPath = "$tools\cygwin" | |
| if (!(Test-Path $cygwinPath)) { | |
| New-Item -ItemType Directory -Force -Path $cygwinPath | Out-Null | |
| } | |
| Invoke-WebRequest -Uri "http://cygwin.com/setup-x86.exe" ` | |
| -OutFile "setup-x86.exe" | |
| $psi = new-object "Diagnostics.ProcessStartInfo" | |
| $psi.FileName = "$tools\setup-x86.exe" | |
| $psi.Arguments = '--quiet-mode --no-shortcuts --only-site --root "' | |
| $psi.Arguments += "$cygwinPath" | |
| $psi.Arguments += '" --site "' | |
| $psi.Arguments += "$env:CYG_MIRROR" | |
| $psi.Arguments += '" --local-package-dir "' | |
| $psi.Arguments += "$cygwinPath\var\cache\setup" | |
| $psi.Arguments += '" --packages "' | |
| $psi.Arguments += "$env:DEPENDENCIES" | |
| $psi.Arguments += '"' | |
| $psi.WorkingDirectory = "$tools" | |
| $proc = [Diagnostics.Process]::Start($psi) | |
| $proc.WaitForExit() | |
| Pop-Location | |
| # To make "$tools\setup-x86.exe" work, add localhost to /etc/hosts as | |
| # cygwin has no /etc/resolv.conf, an alternative is to create | |
| # resolv.conf | |
| # $hostsEdit = "grep -E '^[^#]+localhost' /etc/hosts || {" | |
| # $hostsEdit += " chmod +w /etc/hosts" | |
| # $hostsEdit += "; printf '\r\n127.0.0.1 localhost\r\n' >> /etc/hosts" | |
| # $hostsEdit += "; grep -E '^[^#]+localhost' /etc/hosts; }" | |
| # & "$cygwinPath\bin\bash" -lc "$hostsEdit" | |
| # Check if Cygwin bash works | |
| & "$cygwinPath\bin\bash" -lc ` | |
| "cygcheck -dc cygwin && cmake --version && mvn --version" | |
| # Build | |
| # Debug build for now, as otherwise gcc produces unusable meta server | |
| # executable even with -O1. | |
| $mko = "CMAKE_OPTIONS='-D CMAKE_BUILD_TYPE=Debug" | |
| $mko += " -D QFS_EXTRA_C_OPTIONS=-flax-vector-conversions'" | |
| $mko += ' -j2 --no-print-directory' | |
| & "$cygwinPath\bin\bash" -lc ` | |
| "cd '$cur' ; df -h . ; make $mko tarball" | |
| - name: Upload tarball | |
| if: | | |
| 'refs/heads/master' == github.ref | |
| || startsWith(github.ref, 'refs/tags/') | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-west-2 | |
| shell: pwsh | |
| run: | | |
| aws s3 cp (Get-ChildItem "build\qfs-*.tgz").FullName ` | |
| s3://quantcast-qfs |