Set up CI with Azure Pipelines (#72)

* Set up CI with Azure Pipelines [skip ci] * Fix %PLATFORM% * Add Azure badge * Try to make nimble happy with windows PATH * rename steps [skip ci] * Mixed up posix/windows in pipeline * add missing ARCH: x86 to Linux32 * Linux "32bit" CI was actually 64-bit, remove it from now * Try to workaround path issues in windows * More crazy path workarounds + display gcc version (debug win 32-bit) * keep the bash workaround in * mingw path as separate task * Let's start a fresh day bout against Azure * fix powershell variable interpolation * make all scripts verbose to debug MacOS Go libp2p daemon script * Don't test on Azure macOS, xerrors issue due to Go version: - libp2p/go-libp2p-daemon#139 - status-im/nimbus-eth2#397 - golang/go#32246 * polish + reduce verbosity [skip ci]
vacp2p · Feb 6, 2020 · 8c406fb · 8c406fb
1 parent 88a030d
commit 8c406fb
Show file tree

Hide file tree

Showing 2 changed files with 193 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,8 @@
 
 [![Build Status](https://travis-ci.org/status-im/nim-libp2p.svg?branch=master)](https://travis-ci.org/status-im/nim-libp2p)
 [![Build status](https://ci.appveyor.com/api/projects/status/pqgif5bcie6cp3wi/branch/master?svg=true)](https://ci.appveyor.com/project/nimbus/nim-libp2p/branch/master)
+[![Build Status: Azure](https://img.shields.io/azure-devops/build/nimbus-dev/dc5eed24-3f6c-4c06-8466-3d060abd6c8b/5/master?label=Azure%20%28Linux%2064-bit%2C%20Windows%2032-bit%2F64-bit%2C%20MacOS%2064-bit%29)](https://dev.azure.com/nimbus-dev/nim-libp2p/_build?definitionId=5&branchName=master)
+
 [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -0,0 +1,191 @@
+strategy:
+  maxParallel: 10
+  matrix:
+    # Nim requires enforcing ARCH="x86" and UCPU
+    # for 32-bit targets as it seems like Azure machines are 64-bit
+    # TEST_LANG env variable support TODO
+    Windows_32bit:
+      VM: 'windows-latest'
+      ARCH: x86
+      PLATFORM: x86
+      TEST_LANG: c
+    Windows_64bit:
+      VM: 'windows-latest'
+      PLATFORM: x64
+      TEST_LANG: c
+    # Windows_cpp_64bit:
+    #   VM: 'windows-latest'
+    #   UCPU: amd64
+    #   PLATFORM: x64
+    #   TEST_LANG: cpp
+    Linux_64bit:
+      VM: 'ubuntu-16.04'
+      PLATFORM: x64
+      TEST_LANG: c
+    # Linux_cpp_64bit:
+    #   VM: 'ubuntu-16.04'
+    #   UCPU: amd64
+    #   PLATFORM: x64
+    #   TEST_LANG: cpp
+
+    # TODO this requires gcc -m32 as the Ubuntu image is 64-bit
+    # Linux_32bit:
+    #   VM: 'ubuntu-16.04'
+    #   ARCH: x86
+    #   PLATFORM: x86
+    #   TEST_LANG: c
+
+    # TODO: go version on Azure is problematic,
+    #       we have xerrors package issue
+    # MacOS_64bit:
+    #   VM: 'macOS-10.14'
+    #   PLATFORM: x64
+    #   TEST_LANG: c
+
+pool:
+  vmImage: $(VM)
+
+variables:
+  GOPATH: '$(system.defaultWorkingDirectory)/go'
+  V: 0 # Scripts verbosity, 1 for debugging build scripts
+
+steps:
+  - task: CacheBeta@1
+    displayName: 'cache Nim binaries'
+    inputs:
+      key: NimBinaries | $(Agent.OS) | $(PLATFORM)
+      path: NimBinaries
+
+  - task: CacheBeta@1
+    displayName: 'cache Go libp2p daemon'
+    inputs:
+      key: p2pdCache | $(Agent.OS) | $(PLATFORM)
+      path: p2pdCache
+
+  - task: CacheBeta@1
+    displayName: 'cache MinGW-w64'
+    inputs:
+      key: mingwCache | 8_1_0 | $(PLATFORM)
+      path: mingwCache
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - powershell: |
+      Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1
+    displayName: 'long path support'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+  - bash: |
+      echo "PATH=${PATH}"
+      set -e
+      echo "Installing MinGW-w64"
+      if [[ $PLATFORM == "x86" ]]; then
+        MINGW_FILE="i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z"
+        MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/${MINGW_FILE}"
+        MINGW_DIR="mingw32"
+      else
+        MINGW_FILE="x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z"
+        MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/8.1.0/threads-posix/seh/${MINGW_FILE}"
+        MINGW_DIR="mingw64"
+      fi
+      mkdir -p mingwCache
+      pushd mingwCache
+      if [[ ! -e "$MINGW_FILE" ]]; then
+        rm -f *.7z
+        curl -OLsS "$MINGW_URL"
+      fi
+      7z x -y -bd "$MINGW_FILE" >/dev/null
+      mkdir -p /c/custom
+      mv "$MINGW_DIR" /c/custom/
+      popd
+
+      # Workaround https://developercommunity.visualstudio.com/content/problem/891929/windows-2019-cygheap-base-mismatch-detected-git-ba.html
+      echo "##vso[task.prependpath]/usr/bin"
+      echo "##vso[task.prependpath]/mingw64/bin"
+
+      echo "##vso[task.setvariable variable=MINGW_DIR;]$MINGW_DIR"
+    displayName: 'Install dependencies (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+  - powershell: |
+      # export custom mingw PATH to other tasks
+      echo "##vso[task.prependpath]c:\custom\$(MINGW_DIR)\bin"
+    displayName: 'Mingw PATH (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - bash: |
+      echo "PATH=${PATH}"
+      export ncpu=
+      case '$(Agent.OS)' in
+      'Linux')
+        ncpu=$(nproc)
+        ;;
+      'Darwin')
+        ncpu=$(sysctl -n hw.ncpu)
+        ;;
+      'Windows_NT')
+        ncpu=$NUMBER_OF_PROCESSORS
+        ;;
+      esac
+      [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+      echo "Found ${ncpu} cores"
+      echo "##vso[task.setvariable variable=ncpu;]$ncpu"
+    displayName: 'Detecting number of cores'
+
+  - bash: |
+      echo "PATH=${PATH}"
+
+      # build nim from our own branch - this to avoid the day-to-day churn and
+      # regressions of the fast-paced Nim development while maintaining the
+      # flexibility to apply patches
+      curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+      env MAKE="mingw32-make -j${ncpu}" ARCH_OVERRIDE=$(PLATFORM) bash build_nim.sh Nim csources dist/nimble NimBinaries
+
+    displayName: 'Building Nim (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+  - powershell: |
+      echo "##vso[task.prependpath]$pwd\Nim\bin"
+    displayName: 'Set env variable (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - bash: |
+      echo "PATH=${PATH}"
+
+      # build nim from our own branch - this to avoid the day-to-day churn and
+      # regressions of the fast-paced Nim development while maintaining the
+      # flexibility to apply patches
+      curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+      env MAKE="make -j${ncpu}" ARCH_OVERRIDE=$(PLATFORM) bash build_nim.sh Nim csources dist/nimble NimBinaries
+    displayName: 'Building Nim (POSIX)'
+    condition: ne(variables['Agent.OS'], 'Windows_NT')
+  - bash: |
+      echo "##vso[task.prependpath]$PWD/Nim/bin"
+    displayName: 'Set env variable (Posix)'
+    condition: ne(variables['Agent.OS'], 'Windows_NT')
+
+  - bash: |
+      # install and build go-libp2p-daemon
+      go version
+
+      echo "##vso[task.prependpath]$(GOPATH)/bin"
+      curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh
+      bash build_p2pd.sh p2pdCache
+    displayName: 'Building Go libp2p daemon'
+
+  - bash: |
+      echo "PATH=${PATH}"
+      nimble refresh
+      nimble install -y --depsOnly
+    displayName: 'Building dependencies (Posix)'
+    condition: ne(variables['Agent.OS'], 'Windows_NT')
+  - powershell: |
+      echo $Env:Path
+      nimble refresh
+      nimble install -y --depsOnly
+    displayName: 'Building dependencies (Windows)'
+    condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+  - bash: |
+      echo "PATH=${PATH}"
+      which gcc
+      gcc -v
+
+      nimble test
+    displayName: 'Testing the package'