Skip to content

Commit

Permalink
build: compile FIPS go archives
Browse files Browse the repository at this point in the history
Epic: none
Release note: None
  • Loading branch information
rail committed Jan 26, 2023
1 parent 9caf758 commit 9a98701
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 24 deletions.
36 changes: 29 additions & 7 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -167,17 +167,39 @@ load(

# To point to a mirrored artifact, use:
#
# go_download_sdk(
# name = "go_sdk",
# sdks = {
# "darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"),
# "darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"),
# "freebsd_amd64": ("go1.19.4.freebsd-amd64.tar.gz", "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6"),
# "linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4"),
# "linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552"),
# "windows_amd64": ("go1.19.4.windows-amd64.tar.gz", "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc"),
# },
# urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"],
# version = "1.19.4",
# )
# go_download_sdk(
# name = "go_sdk",
# goos = "darwin",
# goarch = "amd64",
# sdks = {
# "darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"),
# "darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"),
# },
# urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"],
# version = "1.19.4",
# )
go_download_sdk(
name = "go_sdk",
goos = "linux",
goarch = "amd64",
sdks = {
"darwin_amd64": ("go1.19.4.darwin-amd64.tar.gz", "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d"),
"darwin_arm64": ("go1.19.4.darwin-arm64.tar.gz", "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f"),
"freebsd_amd64": ("go1.19.4.freebsd-amd64.tar.gz", "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6"),
"linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4"),
"linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552"),
"windows_amd64": ("go1.19.4.windows-amd64.tar.gz", "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc"),
"linux_amd64": ("go1.19.4.linux-amd64.tar.gz", "77cf5d70867250b5f9b6af751a81b9f96372fbade346722812914ca76a5b0fe4"),
"linux_arm64": ("go1.19.4.linux-arm64.tar.gz", "5a9dfaca3b42eceff7df29954f7f1a53a9416df6dc0dd8664262bbe216692ae5"),
},
urls = ["https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/{}"],
urls = ["https://storage.googleapis.com/public-bazel-artifacts/go-fips/20230126-163017/{}"],
version = "1.19.4",
)

Expand Down
4 changes: 0 additions & 4 deletions build/bazelutil/distdir_files.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -1021,10 +1021,6 @@ DISTDIR_FILES = {
"https://storage.googleapis.com/public-bazel-artifacts/c-deps/20230105-170607/libproj_foreign.windows.20230105-170607.tar.gz": "b819b17740b2a3418d62d2f6db8b245094458180e1e5e301e9f0f4257696fef5",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.darwin-amd64.tar.gz": "6fa1e9087b36fba65625869c885ca9c6f1db734306d8e74836b212248c20d55d",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.darwin-arm64.tar.gz": "bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.freebsd-amd64.tar.gz": "84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.linux-amd64.tar.gz": "e52774e4d6a0bb5bcc5a0f1d11e337929de826b40c99c408283b8854336d9dc4",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.linux-arm64.tar.gz": "8bb193126fea46dca70658b7916b458a22fddb8e37d6deb463f14e10d6f06552",
"https://storage.googleapis.com/public-bazel-artifacts/go/20221219-000617/go1.19.4.windows-amd64.tar.gz": "ced538537d190c03e6e4bffb3b60049794d70f09af7900bd8419b44245b2b5dc",
"https://storage.googleapis.com/public-bazel-artifacts/gomod/github.com/bazelbuild/buildtools/v0.0.0-20200718160251-b1667ff58f71/buildtools-v0.0.0-20200718160251-b1667ff58f71.tar.gz": "a9ef5103739dfb5ed2a5b47ab1654842a89695812e4af09e57d7015a5caf97e0",
"https://storage.googleapis.com/public-bazel-artifacts/java/railroad/rr-1.63-java8.zip": "d2791cd7a44ea5be862f33f5a9b3d40aaad9858455828ebade7007ad7113fb41",
"https://storage.googleapis.com/public-bazel-artifacts/js/node/v16.13.0/node-v16.13.0-darwin-arm64.tar.gz": "46d83fc0bd971db5050ef1b15afc44a6665dee40bd6c1cbaec23e1b40fa49e6d",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash

set -euo pipefail

google_credentials="$GOOGLE_EPHEMERAL_CREDENTIALS"
dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
source "$dir/teamcity-support.sh" # for log_into_gcloud
log_into_gcloud

set -x

this_dir="$(cd "$(dirname "${0}")"; pwd)"
toplevel="$(dirname $(dirname $(dirname $(dirname $this_dir))))"

mkdir -p "${toplevel}"/artifacts
# TODO: pin docker image version
DOCKER_IMAGE=registry.access.redhat.com/ubi8/go-toolset:latest

tc_start_block "Build Go toolchains (linux/amd64)"
docker run --rm -i ${tty-} -v $this_dir/build-and-publish-patched-go:/bootstrap \
-v "${toplevel}"/artifacts:/artifacts \
--user root \
--platform linux/amd64 \
$DOCKER_IMAGE /bootstrap/impl-fips.sh
tc_end_block "Build Go toolchains (linux/amd64)"

tc_start_block "Build Go toolchains (linux/arm64)"
docker run --rm -i ${tty-} -v $this_dir/build-and-publish-patched-go:/bootstrap \
-v "${toplevel}"/artifacts:/artifacts \
--user root \
--platform linux/arm64 \
$DOCKER_IMAGE /bootstrap/impl-fips.sh
tc_end_block "Build Go toolchains (linux/arm64)"

tc_start_block "Publish artifacts"
loc=$(date +%Y%m%d-%H%M%S)
for FILE in `find $root/artifacts -name '*.tar.gz'`; do
BASE=$(basename $FILE)
gsutil cp $FILE gs://public-bazel-artifacts/go-fips/$loc/$BASE
done
tc_end_block "Publish artifacts"

tc_end_block "Print checksums"
sha256sum $root/artifacts/*.tar.gz
tc_start_block "Print checksums"
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash

set -xeuo pipefail

# TODO: create a fork?
GO_FIPS_REPO=https://github.com/golang-fips/go
GO_FIPS_BRANCH=go1.19-fips-release


yum install git golang golang-bin openssl openssl-devel -y
cat /etc/os-release
go version
openssl version
git config --global user.name "golang-fips ci"
git config --global user.email "<>"

mkdir /workspace
cd /workspace
git clone $GO_FIPS_REPO go
cd go
git init
git config --global --add safe.directory /__w/go/go
git checkout $GO_FIPS_BRANCH
sed -i "s/go mod tidy/go mod tidy -go=1.16/g" scripts/create-secondary-patch.sh
./scripts/full-initialize-repo.sh
./scripts/configure-crypto-tests.sh
cd go/src
patch -p2 < /bootstrap/diff.patch
./make.bash -v
cd ../..
GOVERS=$(go/bin/go env GOVERSION)
GOOS=$(go/bin/go env GOOS)
GOARCH=$(go/bin/go env GOARCH)
tar cf - go | gzip -9 > /artifacts/$GOVERS.$GOOS-$GOARCH.tar.gz
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set -xeuo pipefail

# When updating to a new Go version, update all of these variables.
GOVERS=1.19.4
FIPS_PATCHES_RELEASE=1
GOLINK=https://go.dev/dl/go$GOVERS.src.tar.gz
SRCSHASUM=eda74db4ac494800a3e66ee784e495bfbb9b8e535df924a8b01b1a8028b7f368
# We mirror the upstream freebsd because we don't have a cross-compiler targeting it.
Expand All @@ -12,6 +13,8 @@ FREEBSDSHASUM=84489ebb63f1757b79574d7345c647bd40bc6414cecb868c93e24476c2d2b9b6
# We mirror the upstream darwin/arm64 binary because we don't have code-signing yet.
GODARWINARMLINK=https://go.dev/dl/go$GOVERS.darwin-arm64.tar.gz
DARWINARMSHASUM=bb3bc5d7655b9637cfe2b5e90055dee93b0ead50e2ffd091df320d1af1ca853f
FIPS_PATCHES_URL=https://github.com/golang-fips/go/archive/refs/tags/go${GOVERS}-${FIPS_PATCHES_RELEASE}-openssl-fips.tar.gz
FIPS_PATCHES_SHASUM=d87a47deaf4ce70a9f5395c3afdd8f2b69a305638c0eb9ccf95a6b7d5df32f37

apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
Expand All @@ -24,26 +27,30 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
gnupg2 \
golang \
make \
patch \
libssl-dev \
libc6-dev-amd64-cross \
libc6-dev-arm64-cross \
python-is-python3 \
python3 \
python3.8-venv

update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 100 \
--slave /usr/bin/clang++ clang++ /usr/bin/clang++-10

curl -fsSL $GOFREEBSDLINK -o /artifacts/go$GOVERS.freebsd-amd64.tar.gz
echo "$FREEBSDSHASUM /artifacts/go$GOVERS.freebsd-amd64.tar.gz" | sha256sum -c -
curl -fsSL $GODARWINARMLINK -o /artifacts/go$GOVERS.darwin-arm64.tar.gz
echo "$DARWINARMSHASUM /artifacts/go$GOVERS.darwin-arm64.tar.gz" | sha256sum -c -

# curl -fsSL $GOFREEBSDLINK -o /artifacts/go$GOVERS.freebsd-amd64.tar.gz
# echo "$FREEBSDSHASUM /artifacts/go$GOVERS.freebsd-amd64.tar.gz" | sha256sum -c -
# curl -fsSL $GODARWINARMLINK -o /artifacts/go$GOVERS.darwin-arm64.tar.gz
# echo "$DARWINARMSHASUM /artifacts/go$GOVERS.darwin-arm64.tar.gz" | sha256sum -c -
#
# libtapi is required for later versions of MacOSX.
git clone https://github.com/tpoechtrager/apple-libtapi.git
cd apple-libtapi
git checkout a66284251b46d591ee4a0cb4cf561b92a0c138d8
./build.sh
./install.sh
cd ..
rm -rf apple-libtapi
# git clone https://github.com/tpoechtrager/apple-libtapi.git
# cd apple-libtapi
# git checkout a66284251b46d591ee4a0cb4cf561b92a0c138d8
# ./build.sh
# ./install.sh
# cd ..
# rm -rf apple-libtapi

curl -fsSL https://storage.googleapis.com/public-bazel-artifacts/toolchains/crosstool-ng/x86_64/20220711-205918/aarch64-unknown-linux-gnu.tar.gz -o aarch64-unknown-linux-gnu.tar.gz
echo '58407f1f3ed490bd0a0a500b23b88503fbcc25f0f69a0b7f8a3e8e7b9237341b aarch64-unknown-linux-gnu.tar.gz' | sha256sum -c -
Expand All @@ -56,6 +63,12 @@ echo 'b87814aaeed8c68679852029de70cee28f96c352ed31c4c520e7bee55999b1c6 x86_64-w6
echo *.tar.gz | xargs -n1 tar -xzf
rm *.tar.gz

curl -fsSL $FIPS_PATCHES_URL -o fips-patches.tar.gz
echo "$FIPS_PATCHES_SHASUM fips-patches.tar.gz" | sha256sum -c -
mkdir -p /tmp/fips-patches
tar -C /tmp/fips-patches --strip-components 1 -xzf fips-patches.tar.gz
rm fips-patches.tar.gz

curl -fsSL $GOLINK -o golang.tar.gz
echo "$SRCSHASUM golang.tar.gz" | sha256sum -c -
mkdir -p /tmp/go$GOVERS
Expand All @@ -68,6 +81,8 @@ git apply /bootstrap/diff.patch
cd ..

for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do
# TODO: enable me
continue
case $CONFIG in
linux_amd64)
CC_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-cc
Expand All @@ -92,7 +107,7 @@ for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do
if [ $GOOS == darwin ]; then
export LD_LIBRARY_PATH=/x-tools/x86_64-apple-darwin21.2/lib
fi
GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \
GO_GCFLAGS="-I/usr/include" GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \
GOROOT_BOOTSTRAP=$(go env GOROOT) CGO_ENABLED=1 ./make.bash
if [ $GOOS == darwin ]; then
unset LD_LIBRARY_PATH
Expand All @@ -108,4 +123,45 @@ for CONFIG in linux_amd64 linux_arm64 darwin_amd64 windows_amd64; do
rm -rf go/bin
done

# FIPS patches
cd /tmp/go$GOVERS/go
patch -p1 < /tmp/fips-patches/patches/000-initial-setup.patch
patch -p1 < /tmp/fips-patches/patches/001-initial-openssl-for-fips.patch
cd ..

for CONFIG in linux_amd64 linux_arm64; do
case $CONFIG in
linux_amd64)
CC_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-cc
CXX_FOR_TARGET=/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-c++
includes=(/usr/include/openssl /usr/x86_64-linux-gnu/include/bits)
;;
linux_arm64)
CC_FOR_TARGET=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-cc
CXX_FOR_TARGET=/x-tools/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-c++
includes=(/usr/include/openssl /usr/aarch64-linux-gnu/include/bits)
;;
esac
tmp_includes=$(mktemp -d)
for i in "${includes[@]}"; do
ln -s "$i" "$tmp_includes/"
done
find "$tmp_includes" -ls
GOOS=$(echo $CONFIG | cut -d_ -f1)
GOARCH=$(echo $CONFIG | cut -d_ -f2)
cd go/src
CGO_CFLAGS="-I $tmp_includes"
GOOS=$GOOS GOARCH=$GOARCH CC=clang CXX=clang++ CC_FOR_TARGET=$CC_FOR_TARGET CXX_FOR_TARGET=$CXX_FOR_TARGET \
GOROOT_BOOTSTRAP=$(go env GOROOT) CGO_ENABLED=1 ./make.bash
cd ../..
rm -rf /tmp/go$GOVERS/go/pkg/${GOOS}_$GOARCH/cmd
if [ $CONFIG != linux_amd64 ]; then
rm go/bin/go go/bin/gofmt
mv go/bin/${GOOS}_$GOARCH/* go/bin
rm -r go/bin/${GOOS}_$GOARCH
fi
tar cf - go | gzip -9 > /artifacts/go$GOVERS.$GOOS-$GOARCH-fips.tar.gz
rm -rf go/bin "$tmp_includes"
done

sha256sum /artifacts/*.tar.gz

0 comments on commit 9a98701

Please sign in to comment.