Migrate release CI to github (#3406)

* Migrate to github CI * Parallelize GHC builds * Avoid building local packages twice * Prepare for building 9.2.6 * Fix build on darwin M1 9.0.2 * Drop build of windows 9.2.6 due to https://gitlab.haskell.org/ghc/ghc/-/issues/22961 * Drop 9.0.2 for darwin M1 * Fix build on windows by disabling ghc-check pkg ABI check * Use git-archive to create src tarball in upload * Build releases for 9.2.7 * Build src archive in github action * Build on unknown-linux too * Try to build GHC-9.4.4 for unknown linux on Fedora33 * Make sure hls-wrapper binary comes from GHC-8.10.7 --------- Co-authored-by: wz1000 <[email protected]>
haskell · Mar 23, 2023 · 9fe56f1 · 9fe56f1
1 parent 191bda6
commit 9fe56f1
Show file tree

Hide file tree

Showing 20 changed files with 1,729 additions and 915 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -0,0 +1,109 @@
+# release CI for FreeBSD
+compute_engine_instance:
+  image_project: freebsd-org-cloud-dev
+  image: family/freebsd-13-1
+  platform: freebsd
+  disk: 100 # Gb
+
+build_task:
+  timeout_in: 120m
+  only_if: $CIRRUS_TAG != ''
+  env:
+    AWS_ACCESS_KEY_ID: ENCRYPTED[dc5896620ebc12e98e6bbe96f72c5a2fe3785f439b7b2346797355f8d329a4bfd8ef6e58086bfc014be0d914424101cd]
+    AWS_SECRET_ACCESS_KEY: ENCRYPTED[6501cd594aca08c6c67cc679dd6f6d30db0cd44a81cceddebf32bb3d0a37f9af19cd71ddb7169d3f7b284a7829969f9e]
+    S3_HOST: ENCRYPTED[d3fef1b5850e85d80dd1684370b53183df2218f2d36509108a2703371afd9ebd3f9596ad4de52487c15ea29baed606b7]
+    TARBALL_EXT: "tar.xz"
+    ARCH: 64
+    ARTIFACT: "x86_64-freebsd"
+    DISTRO: "na"
+    RUNNER_OS: "FreeBSD"
+    ADD_CABAL_ARGS: "--enable-split-sections"
+    GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
+    CABAL_CACHE_NONFATAL: "yes"
+  matrix:
+    - name: build-ghc-8.10.7
+      env:
+        GHC_VERSION: 8.10.7
+    - name: build-ghc-9.0.2
+      env:
+        GHC_VERSION: 9.0.2
+    - name: build-ghc-9.2.5
+      env:
+        GHC_VERSION: 9.2.5
+    - name: build-ghc-9.2.7
+      env:
+        GHC_VERSION: 9.2.7
+  install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree
+  script:
+    - tzsetup Etc/GMT
+    - adjkerntz -a
+    - bash .github/scripts/build.sh
+    - tar caf out.tar.xz out/ store/
+  binaries_artifacts:
+    path: "out.tar.xz"
+
+
+bindist_task:
+  name: bindist
+  depends_on:
+    - build-ghc-8.10.7
+    - build-ghc-9.0.2
+    - build-ghc-9.2.5
+    - build-ghc-9.2.7
+  timeout_in: 120m
+  only_if: $CIRRUS_TAG != ''
+  env:
+    TARBALL_EXT: "tar.xz"
+    ARCH: 64
+    ARTIFACT: "x86_64-freebsd"
+    DISTRO: "na"
+    RUNNER_OS: "FreeBSD"
+    GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
+  install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree unzip
+  script:
+    - tzsetup Etc/GMT
+    - adjkerntz -a
+
+    - curl -o binaries-8.10.7.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-8.10.7/binaries/out.tar.xz
+    - tar xvf binaries-8.10.7.tar.xz
+    - rm -f binaries-8.10.7.tar.xz
+
+    - curl -o binaries-9.0.2.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-9.0.2/binaries/out.tar.xz
+    - tar xvf binaries-9.0.2.tar.xz
+    - rm -f binaries-9.0.2.tar.xz
+
+    - curl -o binaries-9.2.5.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-9.2.5/binaries/out.tar.xz
+    - tar xvf binaries-9.2.5.tar.xz
+    - rm -f binaries-9.2.5.tar.xz
+
+    - curl -o binaries-9.2.7.tar.xz -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/build-ghc-9.2.7/binaries/out.tar.xz
+    - tar xvf binaries-9.2.7.tar.xz
+    - rm -f binaries-9.2.7.tar.xz
+
+    - bash .github/scripts/bindist.sh
+  bindist_artifacts:
+    path: "./out/*.tar.xz"
+
+test_task:
+  name: test
+  depends_on:
+    - bindist
+  timeout_in: 120m
+  only_if: $CIRRUS_TAG != ''
+  env:
+    TARBALL_EXT: "tar.xz"
+    ARCH: 64
+    ARTIFACT: "x86_64-freebsd"
+    DISTRO: "na"
+    RUNNER_OS: "FreeBSD"
+    GITHUB_WORKSPACE: ${CIRRUS_WORKING_DIR}
+  install_script: pkg install -y hs-cabal-install git bash misc/compat10x misc/compat11x misc/compat12x gmake patchelf tree unzip
+  script:
+    - tzsetup Etc/GMT
+    - adjkerntz -a
+
+    - curl -O -L https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/bindist/bindist.zip
+    - unzip bindist.zip
+
+    - bash .github/scripts/test.sh
+
diff --git a/.github/scripts/bindist.sh b/.github/scripts/bindist.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -eux
+
+. .github/scripts/env.sh
+. .github/scripts/common.sh
+
+# ensure ghcup
+if ! command -v ghcup ; then
+	install_ghcup
+fi
+
+# create tarball/zip
+case "${TARBALL_EXT}" in
+    zip)
+		HLS_VERSION="$(grep '^version:' haskell-language-server.cabal | awk '{ print $2 }')"
+		(
+			cd "$CI_PROJECT_DIR/out/${ARTIFACT}"
+			zip "$CI_PROJECT_DIR/out/haskell-language-server-${HLS_VERSION}-${ARTIFACT}.zip" haskell-language-server-*
+		)
+        ;;
+    tar.xz)
+		# we need to control the order, so the hls wrapper binary is installed
+		# from the oldest version in the list
+		: "${GHCS:="$(cd "$CI_PROJECT_DIR/out/${ARTIFACT}" && rm -f ./*.json && for ghc in * ; do printf "%s\n" "$ghc" ; done | sort -r | tr '\n' ' ')"}"
+		emake --version
+		emake GHCUP=ghcup ARTIFACT="${ARTIFACT}" GHCS="${GHCS}" bindist
+		emake GHCUP=ghcup ARTIFACT="${ARTIFACT}"                bindist-tar
+		emake GHCUP=ghcup                        GHCS="${GHCS}" clean-ghcs
+        ;;
+    *)
+        fail "Unknown TARBALL_EXT: ${TARBALL_EXT}"
+        ;;
+esac
diff --git a/.github/scripts/brew.sh b/.github/scripts/brew.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -eux
+
+. .github/scripts/env.sh
+
+if [ -e "$HOME/.brew" ] ; then
+	(
+	cd "$HOME/.brew"
+	git fetch --depth 1
+	git reset --hard origin/master
+	)
+else
+	git clone --depth=1 https://github.com/Homebrew/brew "$HOME/.brew"
+fi
+export PATH="$HOME/.brew/bin:$HOME/.brew/sbin:$PATH"
+
+mkdir -p $CI_PROJECT_DIR/.brew_cache
+export HOMEBREW_CACHE=$CI_PROJECT_DIR/.brew_cache
+mkdir -p $CI_PROJECT_DIR/.brew_logs
+export HOMEBREW_LOGS=$CI_PROJECT_DIR/.brew_logs
+mkdir -p /private/tmp/.brew_tmp
+export HOMEBREW_TEMP=/private/tmp/.brew_tmp
+
+#brew update
+brew install ${1+"$@"}
+
diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -eux
+
+. .github/scripts/env.sh
+. .github/scripts/common.sh
+
+uname -a
+uname -p
+uname
+pwd
+env
+
+# ensure ghcup
+if ! command -v ghcup ; then
+	install_ghcup
+fi
+
+# ensure cabal-cache
+download_cabal_cache "$HOME/.local/bin/cabal-cache"
+
+
+# build
+ecabal update
+ghcup install ghc "${GHC_VERSION}"
+ghcup set ghc "${GHC_VERSION}"
+"ghc-${GHC_VERSION}" --info
+"ghc" --info
+
+mkdir -p "$CI_PROJECT_DIR/out/${ARTIFACT}"
+mkdir -p "$CI_PROJECT_DIR/out/plan.json"
+
+case "$(uname)" in
+    MSYS_*|MINGW*)
+		args=( -O2 -w "ghc-$GHC_VERSION" --project-file cabal.project --disable-profiling --disable-tests --enable-executable-stripping ${ADD_CABAL_ARGS})
+
+		# Shorten binary names
+		# due to MAX_PATH issues on windows
+		sed -i.bak -e 's/haskell-language-server/hls/g' \
+			   -e 's/haskell_language_server/hls/g' \
+			   haskell-language-server.cabal cabal.project
+		sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
+			   src/**/*.hs exe/*.hs
+
+		# shellcheck disable=SC2068
+		build_with_cache ${args[@]} exe:hls exe:hls-wrapper
+		cp dist-newstyle/cache/plan.json "$CI_PROJECT_DIR/out/plan.json/${ARTIFACT}-ghc-${GHC_VERSION}-plan.json"
+
+		# shellcheck disable=SC2068
+		cp "$(cabal list-bin -v0 ${args[@]} exe:hls)" "$CI_PROJECT_DIR/out/${ARTIFACT}/haskell-language-server-${GHC_VERSION}${ext}"
+		# shellcheck disable=SC2068
+		cp "$(cabal list-bin -v0 ${args[@]} exe:hls-wrapper)" "$CI_PROJECT_DIR/out/${ARTIFACT}/haskell-language-server-wrapper${ext}"
+        ;;
+	*)
+		sed -i.bak -e '/DELETE MARKER FOR CI/,/END DELETE/d' cabal.project # see comment in cabal.project
+		emake --version
+		emake GHCUP=ghcup CABAL_CACHE_BIN=cabal-cache.sh S3_HOST="${S3_HOST}" S3_KEY="${ARTIFACT}" GHC_VERSION="${GHC_VERSION}" hls-ghc
+        ;;
+esac
+
+
diff --git a/.github/scripts/cabal-cache.sh b/.github/scripts/cabal-cache.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+case "$(uname -s)" in
+	MSYS_*|MINGW*)
+		ext=".exe"
+		;;
+	*)
+		ext=""
+	   ;;
+esac
+
+if [ "${CABAL_CACHE_DISABLE}" = "yes" ] ; then
+	echo "cabal-cache disabled (CABAL_CACHE_DISABLE set)"
+elif [ "${CABAL_CACHE_NONFATAL}" = "yes" ] ; then
+	time "cabal-cache${ext}" "$@" || echo "cabal-cache failed (CABAL_CACHE_NONFATAL set)"
+else
+	time "cabal-cache${ext}" "$@"
+	exit $?
+fi
+