From 9a987010d01d05b04046361b040ab362eca82fb2 Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Thu, 19 Jan 2023 11:50:56 -0500 Subject: [PATCH] build: compile FIPS go archives Epic: none Release note: None --- WORKSPACE | 36 ++++++-- build/bazelutil/distdir_files.bzl | 4 - .../build-and-publish-patched-go-fips.sh | 45 ++++++++++ .../build-and-publish-patched-go/impl-fips.sh | 34 ++++++++ .../build-and-publish-patched-go/impl.sh | 82 ++++++++++++++++--- 5 files changed, 177 insertions(+), 24 deletions(-) create mode 100755 build/teamcity/internal/release/build-and-publish-patched-go-fips.sh create mode 100755 build/teamcity/internal/release/build-and-publish-patched-go/impl-fips.sh diff --git a/WORKSPACE b/WORKSPACE index 7636c9729f7f..1689d79fdc81 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -167,17 +167,39 @@ load( # To point to a mirrored artifact, use: # +# go_download_sdk( +# name = "go_sdk", +# sdks = { +# "darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"), +# "darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"), +# "freebsd_amd64": ("go1.19.4.freebsd-amd64.tar.gz", "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6"), +# "linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4"), +# "linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552"), +# "windows_amd64": ("go1.19.4.windows-amd64.tar.gz", "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc"), +# }, +# urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"], +# version = "1.19.4", +# ) +# go_download_sdk( +# name = "go_sdk", +# goos = "darwin", +# goarch = "amd64", +# sdks = { +# "darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"), +# "darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"), +# }, +# urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"], +# version = "1.19.4", +# ) go_download_sdk( name = "go_sdk", + goos = "linux", + goarch = "amd64", sdks = { - "darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"), - "darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"), - "freebsd_amd64": ("go1.19.4.freebsd-amd64.tar.gz", "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6"), - "linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4"), - "linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552"), - "windows_amd64": ("go1.19.4.windows-amd64.tar.gz", "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc"), + "linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "77cf5d70867250b5f9b6af751a81b9f96372fbade346722812914ca76a5b0fe4"), + "linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "5a9dfaca3b42eceff7df29954f7f1a53a9416df6dc0dd8664262bbe216692ae5"), }, - urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"], + urls = ["https://storage.googleapis.com/public-bazel-artifacts/go-fips/20230126-163017/{}"], version = "1.19.4", ) diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index 4c3dbf83af2d..1d9d4e830756 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -1021,10 +1021,6 @@ DISTDIR_FILES = { "https://storage.googleapis.com/public-bazel-artifacts/c-deps/20230105-170607/libproj_foreign.windows.20230105-170607.tar.gz": "b819b17740b2a3418d62d2f6db8b245094458180e1e5e301e9f0f4257696fef5", "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.darwin-amd64.tar.gz": "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d", "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.darwin-arm64.tar.gz": "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f", - "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.freebsd-amd64.tar.gz": "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6", - "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.linux-amd64.tar.gz": "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4", - "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.linux-arm64.tar.gz": "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552", - "https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.windows-amd64.tar.gz": "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc", "https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v0.0.0-20200718160251-b1667ff58f71/buildtools-v0.0.0-20200718160251-b1667ff58f71.tar.gz": "a9ef5103739dfb5ed2a5b47ab1654842a89695812e4af09e57d7015a5caf97e0", "https://storage.googleapis.com/public-bazel-artifacts/java/railroad/rr-1.63-java8.zip": "d2791cd7a44ea5be862f33f5a9b3d40aaad9858455828ebade7007ad7113fb41", "https://storage.googleapis.com/public-bazel-artifacts/js/node/v16.13.0/node-v16.13.0-darwin-arm64.tar.gz": "46d83fc0bd971db5050ef1b15afc44a6665dee40bd6c1cbaec23e1b40fa49e6d", diff --git a/build/teamcity/internal/release/build-and-publish-patched-go-fips.sh b/build/teamcity/internal/release/build-and-publish-patched-go-fips.sh new file mode 100755 index 000000000000..3a9d766e1076 --- /dev/null +++ b/build/teamcity/internal/release/build-and-publish-patched-go-fips.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS" +dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" +source "$dir/teamcity-support.sh" # for log_into_gcloud +log_into_gcloud + +set -x + +this_dir="$(cd "$(dirname "${0}")"; pwd)" +toplevel="$(dirname $(dirname $(dirname $(dirname $this_dir))))" + +mkdir -p "${toplevel}"/artifacts +# TODO: pin docker image version +DOCKER_IMAGE=registry.access.redhat.com/ubi8/go-toolset:latest + +tc_start_block "Build Go toolchains (linux/amd64)" +docker run --rm -i ${tty-} -v $this_dir/build-and-publish-patched-go:/bootstrap \ + -v "${toplevel}"/artifacts:/artifacts \ + --user root \ + --platform linux/amd64 \ + $DOCKER_IMAGE /bootstrap/impl-fips.sh +tc_end_block "Build Go toolchains (linux/amd64)" + +tc_start_block "Build Go toolchains (linux/arm64)" +docker run --rm -i ${tty-} -v $this_dir/build-and-publish-patched-go:/bootstrap \ + -v "${toplevel}"/artifacts:/artifacts \ + --user root \ + --platform linux/arm64 \ + $DOCKER_IMAGE /bootstrap/impl-fips.sh +tc_end_block "Build Go toolchains (linux/arm64)" + +tc_start_block "Publish artifacts" +loc=$(date +%Y%m%d-%H%M%S) +for FILE in `find $root/artifacts -name '*.tar.gz'`; do + BASE=$(basename $FILE) + gsutil cp $FILE gs://public-bazel-artifacts/go-fips/$loc/$BASE +done +tc_end_block "Publish artifacts" + +tc_end_block "Print checksums" +sha256sum $root/artifacts/*.tar.gz +tc_start_block "Print checksums" diff --git a/build/teamcity/internal/release/build-and-publish-patched-go/impl-fips.sh b/build/teamcity/internal/release/build-and-publish-patched-go/impl-fips.sh new file mode 100755 index 000000000000..14b6aa110f9b --- /dev/null +++ b/build/teamcity/internal/release/build-and-publish-patched-go/impl-fips.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -xeuo pipefail + +# TODO: create a fork? +GO_FIPS_REPO=https://github.com/golang-fips/go +GO_FIPS_BRANCH=go1.19-fips-release + + +yum install git golang golang-bin openssl openssl-devel -y +cat /etc/os-release +go version +openssl version +git config --global user.name "golang-fips ci" +git config --global user.email "<>" + +mkdir /workspace +cd /workspace +git clone $GO_FIPS_REPO go +cd go +git init +git config --global --add safe.directory /__w/go/go +git checkout $GO_FIPS_BRANCH +sed -i "s/go mod tidy/go mod tidy -go=1.16/g" scripts/create-secondary-patch.sh +./scripts/full-initialize-repo.sh +./scripts/configure-crypto-tests.sh +cd go/src +patch -p2 < /bootstrap/diff.patch +./make.bash -v +cd ../.. +GOVERS=$(go/bin/go env GOVERSION) +GOOS=$(go/bin/go env GOOS) +GOARCH=$(go/bin/go env GOARCH) +tar cf - go | gzip -9 > /artifacts/$GOVERS.$GOOS-$GOARCH.tar.gz diff --git a/build/teamcity/internal/release/build-and-publish-patched-go/impl.sh b/build/teamcity/internal/release/build-and-publish-patched-go/impl.sh index a31fbaeafaea..35f05b5ecc25 100755 --- a/build/teamcity/internal/release/build-and-publish-patched-go/impl.sh +++ b/build/teamcity/internal/release/build-and-publish-patched-go/impl.sh @@ -4,6 +4,7 @@ set -xeuo pipefail # When updating to a new Go version, update all of these variables. GOVERS=1.19.4 +FIPS_PATCHES_RELEASE=1 GOLINK=https://go.dev/dl/go$GOVERS.src.tar.gz SRCSHASUM=eda74db4ac494800a3e66ee784e495bfbb9b8e535df924a8b01b1a8028b7f368 # We mirror the upstream freebsd because we don't have a cross-compiler targeting it. @@ -12,6 +13,8 @@ FREEBSDSHASUM=84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6 # We mirror the upstream darwin/arm64 binary because we don't have code-signing yet. GODARWINARMLINK=https://go.dev/dl/go$GOVERS.darwin-arm64.tar.gz DARWINARMSHASUM=bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f +FIPS_PATCHES_URL=https://github.com/golang-fips/go/archive/refs/tags/go${GOVERS}-${FIPS_PATCHES_RELEASE}-openssl-fips.tar.gz +FIPS_PATCHES_SHASUM=d87a47deaf4ce70a9f5395c3afdd8f2b69a305638c0eb9ccf95a6b7d5df32f37 apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -24,6 +27,10 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ gnupg2 \ golang \ make \ + patch \ + libssl-dev \ + libc6-dev-amd64-cross \ + libc6-dev-arm64-cross \ python-is-python3 \ python3 \ python3.8-venv @@ -31,19 +38,19 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \ --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 -curl -fsSL $GOFREEBSDLINK -o /artifacts/go$GOVERS.freebsd-amd64.tar.gz -echo "$FREEBSDSHASUM /artifacts/go$GOVERS.freebsd-amd64.tar.gz" | sha256sum -c - -curl -fsSL $GODARWINARMLINK -o /artifacts/go$GOVERS.darwin-arm64.tar.gz -echo "$DARWINARMSHASUM /artifacts/go$GOVERS.darwin-arm64.tar.gz" | sha256sum -c - - +# curl -fsSL $GOFREEBSDLINK -o /artifacts/go$GOVERS.freebsd-amd64.tar.gz +# echo "$FREEBSDSHASUM /artifacts/go$GOVERS.freebsd-amd64.tar.gz" | sha256sum -c - +# curl -fsSL $GODARWINARMLINK -o /artifacts/go$GOVERS.darwin-arm64.tar.gz +# echo "$DARWINARMSHASUM /artifacts/go$GOVERS.darwin-arm64.tar.gz" | sha256sum -c - +# # libtapi is required for later versions of MacOSX. -git clone https://github.com/tpoechtrager/apple-libtapi.git -cd apple-libtapi -git checkout a66284251b46d591ee4a0cb4cf561b92a0c138d8 -./build.sh -./install.sh -cd .. -rm -rf apple-libtapi +# git clone https://github.com/tpoechtrager/apple-libtapi.git +# cd apple-libtapi +# git checkout a66284251b46d591ee4a0cb4cf561b92a0c138d8 +# ./build.sh +# ./install.sh +# cd .. +# rm -rf apple-libtapi curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/x86_64/20220711-205918/aarch64-unknown-linux-gnu.tar.gz -o aarch64-unknown-linux-gnu.tar.gz echo '58407f1f3ed490bd0a0a500b23b88503fbcc25f0f69a0b7f8a3e8e7b9237341b aarch64-unknown-linux-gnu.tar.gz' | sha256sum -c - @@ -56,6 +63,12 @@ echo 'b87814aaeed8c68679852029de70cee28f96c352ed31c4c520e7bee55999b1c6 x86_64-w6 echo *.tar.gz | xargs -n1 tar -xzf rm *.tar.gz +curl -fsSL $FIPS_PATCHES_URL -o fips-patches.tar.gz +echo "$FIPS_PATCHES_SHASUM fips-patches.tar.gz" | sha256sum -c - +mkdir -p /tmp/fips-patches +tar -C /tmp/fips-patches --strip-components 1 -xzf fips-patches.tar.gz +rm fips-patches.tar.gz + curl -fsSL $GOLINK -o golang.tar.gz echo "$SRCSHASUM golang.tar.gz" | sha256sum -c - mkdir -p /tmp/go$GOVERS @@ -68,6 +81,8 @@ git apply /bootstrap/diff.patch cd .. for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do + # TODO: enable me + continue case $CONFIG in linux_amd64) CC_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-cc @@ -92,7 +107,7 @@ for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do if [ $GOOS == darwin ]; then export LD_LIBRARY_PATH=/x-tools/x86_64-apple-darwin21.2/lib fi - GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \ + GO_GCFLAGS="-I/usr/include" GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \ GOROOT_BOOTSTRAP=$(go env GOROOT) CGO_ENABLED=1 ./make.bash if [ $GOOS == darwin ]; then unset LD_LIBRARY_PATH @@ -108,4 +123,45 @@ for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do rm -rf go/bin done +# FIPS patches +cd /tmp/go$GOVERS/go +patch -p1 < /tmp/fips-patches/patches/000-initial-setup.patch +patch -p1 < /tmp/fips-patches/patches/001-initial-openssl-for-fips.patch +cd .. + +for CONFIG in linux_amd64 linux_arm64; do + case $CONFIG in + linux_amd64) + CC_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-cc + CXX_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-c++ + includes=(/usr/include/openssl /usr/x86_64-linux-gnu/include/bits) + ;; + linux_arm64) + CC_FOR_TARGET=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-cc + CXX_FOR_TARGET=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-c++ + includes=(/usr/include/openssl /usr/aarch64-linux-gnu/include/bits) + ;; + esac + tmp_includes=$(mktemp -d) + for i in "${includes[@]}"; do + ln -s "$i" "$tmp_includes/" + done + find "$tmp_includes" -ls + GOOS=$(echo $CONFIG | cut -d_ -f1) + GOARCH=$(echo $CONFIG | cut -d_ -f2) + cd go/src + CGO_CFLAGS="-I $tmp_includes" + GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \ + GOROOT_BOOTSTRAP=$(go env GOROOT) CGO_ENABLED=1 ./make.bash + cd ../.. + rm -rf /tmp/go$GOVERS/go/pkg/${GOOS}_$GOARCH/cmd + if [ $CONFIG != linux_amd64 ]; then + rm go/bin/go go/bin/gofmt + mv go/bin/${GOOS}_$GOARCH/* go/bin + rm -r go/bin/${GOOS}_$GOARCH + fi + tar cf - go | gzip -9 > /artifacts/go$GOVERS.$GOOS-$GOARCH-fips.tar.gz + rm -rf go/bin "$tmp_includes" +done + sha256sum /artifacts/*.tar.gz