From ff0b721e5445527fc9d36b4425f544f02ee298e3 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Wed, 12 Aug 2020 23:54:36 +0000 Subject: [PATCH 01/33] run-docker-build-from-artifacts Trigger docker build + release from build artifacts --- buildkite/src/Jobs/Artifact.dhall | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Artifact.dhall b/buildkite/src/Jobs/Artifact.dhall index fd17bcbd777..38938b4078b 100644 --- a/buildkite/src/Jobs/Artifact.dhall +++ b/buildkite/src/Jobs/Artifact.dhall @@ -12,6 +12,9 @@ let OpamInit = ../Command/OpamInit.dhall let Summon = ../Command/Summon/Type.dhall let Size = ../Command/Size.dhall let Libp2p = ../Command/Libp2pHelperBuild.dhall +let DockerArtifact = ../Command/DockerArtifact.dhall + +let dependsOn = [ { name = "Artifact", key = "artifacts-build" } ] in Pipeline.build @@ -39,6 +42,7 @@ Pipeline.build label = "Build artifacts", key = "artifacts-build", target = Size.Large - } + }, + DockerArtifact.generateStep dependsOn ] } From a5aa3bdbe07fa8dc4c7701f19729a1f77c11d08e Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 00:23:30 +0000 Subject: [PATCH 02/33] run-docker-build-from-artifacts Add CODA_VERSION to DOCKER_DEPLOY_ENV --- buildkite/scripts/publish-deb.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 4547a149706..6679cb36588 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -33,6 +33,9 @@ set -x ${DEBS3} --codename "${CODENAME}" "${DEBS}" echo "Exporting Variables: " # Export Variables for Downstream Steps + +source ./DOCKER_DEPLOY_ENV +echo "export CODA_VERSION=$CODA_DEB_VERSION" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV set +x From f9fe08ba28062b2e2a85a07087941965c0b2e4ab Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 00:57:42 +0000 Subject: [PATCH 03/33] run-docker-build-from-artifacts Use rebuild-deb to make a more complete and specific DOCKER_DEPLOY_ENV --- buildkite/scripts/publish-deb.sh | 33 +++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 6679cb36588..64a33f21606 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -31,11 +31,38 @@ esac echo "Publishing debs: ${DEBS}" set -x ${DEBS3} --codename "${CODENAME}" "${DEBS}" +set +x echo "Exporting Variables: " -# Export Variables for Downstream Steps -source ./DOCKER_DEPLOY_ENV -echo "export CODA_VERSION=$CODA_DEB_VERSION" >> ./DOCKER_DEPLOY_ENV +GITHASH=$(git rev-parse --short=7 HEAD) +GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) +GITTAG=$(git describe --abbrev=0) + + +# Identify All Artifacts by Branch and Git Hash +set +u +PVKEYHASH=$(./_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) + +PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" + +BUILD_NUM=${BUILDKITE_BUILD_NUM} +BUILD_URL=${BUILDKITE_BUILD_URL} + +if [ "$GITBRANCH" == "master" ]; then + VERSION="${GITTAG}-${GITHASH}" +else + VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" +fi + +set -x +# Export variables for use with downstream steps +echo "export CODA_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_DEB_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_BRANCH=$GITBRANCH" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV set +x + From 292e0fd03c7f20ec4a38b7901e5c4ceda646515c Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 01:35:25 +0000 Subject: [PATCH 04/33] run-docker-build-from-artifacts Simplify extra-args --- buildkite/scripts/docker-artifact.sh | 6 +++--- scripts/release-docker.sh | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 71c90a079f5..35e6c2f985e 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -3,7 +3,7 @@ set -eo pipefail source DOCKER_DEPLOY_ENV -extra_args="--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" - echo "--- Build/Release docker artifact" -scripts/release-docker.sh --service $CODA_SERVICE --version $CODA_VERSION --extra-args "${extra_args}" +scripts/release-docker.sh --service $CODA_SERVICE --version $CODA_VERSION \ +--extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" + diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 0ae612c23d4..08bf0234c5f 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -4,7 +4,8 @@ # we have to trustlist and configure image builds individually because each one is going to be slightly different. # This is needed as opposed to trusting the structure of the each project to be consistent for every deployable. -set -e+x +set -e +set +x CLEAR='\033[0m' RED='\033[0;31m' From 9804d2495440935d280087b420f0b65cd340fce9 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 01:42:24 +0000 Subject: [PATCH 05/33] run-docker-build-from-artifacts Quote version and service just in case --- buildkite/scripts/docker-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 35e6c2f985e..a6a4e399d10 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -4,6 +4,6 @@ set -eo pipefail source DOCKER_DEPLOY_ENV echo "--- Build/Release docker artifact" -scripts/release-docker.sh --service $CODA_SERVICE --version $CODA_VERSION \ +scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}" \ --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" From b20dde4a56925121525bdfdbc570e2c57a4d1778 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:07:23 +0000 Subject: [PATCH 06/33] run-docker-build-from-artifacts Be more verbose and add rosetta support --- buildkite/scripts/docker-artifact.sh | 5 +++-- scripts/release-docker.sh | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index a6a4e399d10..59e252e35c4 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -1,9 +1,10 @@ #!/bin/bash set -eo pipefail +set +x source DOCKER_DEPLOY_ENV echo "--- Build/Release docker artifact" -scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}" \ ---extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" +scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}" --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" +scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}" --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 08bf0234c5f..8d8d890a6d0 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -53,6 +53,10 @@ coda-daemon) coda-demo) DOCKERFILE_PATH="dockerfiles/Dockerfile-coda-demo" DOCKER_CONTEXT="." +coda-rosetta) + DOCKERFILE_PATH="dockerfiles/Dockerfile-rosetta" + mkdir some_empty_dir + DOCKER_CONTEXT="./some_empty_dir" ;; *) esac From 7b61c1e66a7d8c3f446b3579d8d87999b168a309 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:08:03 +0000 Subject: [PATCH 07/33] run-docker-build-from-artifacts Fix bash syntax --- scripts/release-docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 8d8d890a6d0..af8c84d8b30 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -53,6 +53,7 @@ coda-daemon) coda-demo) DOCKERFILE_PATH="dockerfiles/Dockerfile-coda-demo" DOCKER_CONTEXT="." + ;; coda-rosetta) DOCKERFILE_PATH="dockerfiles/Dockerfile-rosetta" mkdir some_empty_dir From 0312bff0d533596b58178edb6fd51792617c0e82 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:17:25 +0000 Subject: [PATCH 08/33] run-docker-build-from-artifacts Fix flags for docker builds from publish-deb --- buildkite/scripts/docker-artifact.sh | 6 ++++-- buildkite/scripts/publish-deb.sh | 1 + scripts/release-docker.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 59e252e35c4..709a531a8aa 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -5,6 +5,8 @@ set +x source DOCKER_DEPLOY_ENV echo "--- Build/Release docker artifact" -scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}" --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" -scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}" --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" +scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}"\ + --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" +scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ + --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH}" diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 64a33f21606..880037d17ce 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -56,6 +56,7 @@ fi set -x # Export variables for use with downstream steps +echo "export CODA_SERVICE=coda-daemon" >> ./DOCKER_DEPLOY_ENV echo "export CODA_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index af8c84d8b30..08a1ea038e4 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -10,7 +10,7 @@ set +x CLEAR='\033[0m' RED='\033[0;31m' # Array of valid service names -VALID_SERVICES=('coda-daemon' 'bot' 'coda-demo') +VALID_SERVICES=('coda-daemon' 'bot' 'coda-demo' 'coda-rosetta') function usage() { if [ -n "$1" ]; then From 9fcdfb246944b5ebec7396ae02dde36ae4a43c63 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:42:47 +0000 Subject: [PATCH 09/33] run-docker-build-from-artifacts Remove + from docker tag as its invalid for tags --- buildkite/scripts/publish-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 880037d17ce..670ff274346 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -51,7 +51,7 @@ BUILD_URL=${BUILDKITE_BUILD_URL} if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" else - VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" + VERSION="${GITTAG}--${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" fi set -x From 99dbb24ad6e24cc6070c14c04993d0165399ef4b Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:47:51 +0000 Subject: [PATCH 10/33] run-docker-build-from-artifacts Sanitize docker image tags --- buildkite/scripts/publish-deb.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 670ff274346..ae347b02e16 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -50,14 +50,16 @@ BUILD_URL=${BUILDKITE_BUILD_URL} if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" + TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" else - VERSION="${GITTAG}--${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" + VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" + TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!; s!_!-!g')" fi set -x # Export variables for use with downstream steps echo "export CODA_SERVICE=coda-daemon" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_VERSION=${GITTAG}-${GITHASH}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV From 51ca606cebf82dd1c072334979b5eecc08d34990 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 02:48:41 +0000 Subject: [PATCH 11/33] run-docker-build-from-artifacts Fix CODA_VERSION --- buildkite/scripts/publish-deb.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index ae347b02e16..acacb8172d3 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -50,17 +50,17 @@ BUILD_URL=${BUILDKITE_BUILD_URL} if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" - TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" + DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" else VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" - TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!; s!_!-!g')" + DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!; s!_!-!g')" fi set -x # Export variables for use with downstream steps echo "export CODA_SERVICE=coda-daemon" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_VERSION=${GITTAG}-${GITHASH}" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_DEB_VERSION=$VERSION" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_VERSION=${DOCKER_TAG}" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_DEB_VERSION=${VERSION}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_BRANCH=$GITBRANCH" >> ./DOCKER_DEPLOY_ENV From dded5c9ff521d8e70475a2411f3f6e3a5aa69c7e Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 03:17:41 +0000 Subject: [PATCH 12/33] run-docker-build-from-artifacts Be more selective about building rosetta and more explicit when its happenning --- buildkite/scripts/docker-artifact.sh | 9 ++++++--- buildkite/scripts/publish-deb.sh | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 709a531a8aa..420bdbc1829 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -4,9 +4,12 @@ set +x source DOCKER_DEPLOY_ENV -echo "--- Build/Release docker artifact" +echo "--- Build/Release docker artifact for coda-daemon" scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}"\ --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" -scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ - --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH}" +if [[ $CODA_BUILD_ROSETTA ]]; then + echo "--- Build/Release coda-rosetta to docker hub" + scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ + --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH}" +fi diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index acacb8172d3..ff367945442 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -48,12 +48,14 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -if [ "$GITBRANCH" == "master" ]; then +if [[ "$GITBRANCH" == "master" ]]; then VERSION="${GITTAG}-${GITHASH}" DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" + BUILD_ROSETTA=true else VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!; s!_!-!g')" + [[ "${GITBRANCH}" == "develop" ]] || [[ "${GITBRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true fi set -x @@ -67,5 +69,6 @@ echo "export CODA_GIT_BRANCH=$GITBRANCH" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_BUILD_ROSETTA=$ROSETTA" >> ./DOCKER_DEPLOY_ENV set +x From ef3a744850d76016e2daf9274dccd7d22092436e Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 03:24:04 +0000 Subject: [PATCH 13/33] run-docker-build-from-artifacts Use buildkite_branch --- buildkite/scripts/publish-deb.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index ff367945442..7efbaaf3599 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -35,7 +35,7 @@ set +x echo "Exporting Variables: " GITHASH=$(git rev-parse --short=7 HEAD) -GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) +#GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) GITTAG=$(git describe --abbrev=0) @@ -48,14 +48,14 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -if [[ "$GITBRANCH" == "master" ]]; then +if [[ "$BUILDKITE_BRANCH" == "master" ]]; then VERSION="${GITTAG}-${GITHASH}" DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" BUILD_ROSETTA=true else - VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" - DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!; s!_!-!g')" - [[ "${GITBRANCH}" == "develop" ]] || [[ "${GITBRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true + VERSION="${GITTAG}+${BUILD_NUM}-${BUILDKITE_BRANCH}-${GITHASH}-PV${PVKEYHASH}" + DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!; s!_!-!g')" + [[ "${GITBRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true fi set -x From 2350976ccb2693a10c48a3e365a8126b54f4bcf2 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 03:58:24 +0000 Subject: [PATCH 14/33] run-docker-build-from-artifacts Also use buildkite_branch for deb builds --- scripts/rebuild-deb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index fa5ac23fc0c..adb947ad635 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -19,6 +19,7 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} +[[ "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g' ) if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" From 064d6073bd372c720ec550bc7d4c3f432ab7d316 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 04:10:09 +0000 Subject: [PATCH 15/33] run-docker-build-from-artifacts Use buildkite_branch more consistently --- buildkite/scripts/publish-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 7efbaaf3599..905b00ef6d3 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -55,7 +55,7 @@ if [[ "$BUILDKITE_BRANCH" == "master" ]]; then else VERSION="${GITTAG}+${BUILD_NUM}-${BUILDKITE_BRANCH}-${GITHASH}-PV${PVKEYHASH}" DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!; s!_!-!g')" - [[ "${GITBRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true + [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true fi set -x From aea40d955c41bee75c1048974099ba90df791b5f Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 04:26:08 +0000 Subject: [PATCH 16/33] rosetta/test-build-rosetta-dockerfile-in-ci Use buildkite_branch for CODA_GIT_BRANCH --- buildkite/scripts/publish-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 905b00ef6d3..71488f07de6 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -65,7 +65,7 @@ echo "export CODA_VERSION=${DOCKER_TAG}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_VERSION=${VERSION}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_GIT_BRANCH=$GITBRANCH" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_BRANCH=$BUILDKITE_BRANCH" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV From fbaa4cf2010e9451cc0950d93b350e3ec9abd06f Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 04:47:47 +0000 Subject: [PATCH 17/33] rosetta/test-build-rosetta-dockerfile-in-ci Fix bash syntax --- buildkite/scripts/publish-deb.sh | 4 ++-- scripts/rebuild-deb.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 71488f07de6..d26ac627449 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -54,7 +54,7 @@ if [[ "$BUILDKITE_BRANCH" == "master" ]]; then BUILD_ROSETTA=true else VERSION="${GITTAG}+${BUILD_NUM}-${BUILDKITE_BRANCH}-${GITHASH}-PV${PVKEYHASH}" - DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!; s!_!-!g')" + DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!g; s!_!-!g')" [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true fi @@ -65,7 +65,7 @@ echo "export CODA_VERSION=${DOCKER_TAG}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_VERSION=${VERSION}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_GIT_BRANCH=$BUILDKITE_BRANCH" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_BRANCH=${BUILDKITE_BRANCH}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index adb947ad635..f85d7815c0c 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -19,7 +19,7 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -[[ "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g' ) +[[ -z "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g' ) if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" From 543cb2a4fcfa58e9b854d9ff068e19d83ff20217 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 04:50:38 +0000 Subject: [PATCH 18/33] rosetta/test-build-rosetta-dockerfile-in-ci Dont escape buildkite_branch in rebuild-deb --- buildkite/scripts/docker-artifact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/docker-artifact.sh b/buildkite/scripts/docker-artifact.sh index 420bdbc1829..59e0159c753 100755 --- a/buildkite/scripts/docker-artifact.sh +++ b/buildkite/scripts/docker-artifact.sh @@ -8,7 +8,7 @@ echo "--- Build/Release docker artifact for coda-daemon" scripts/release-docker.sh --service "${CODA_SERVICE}" --version "${CODA_VERSION}"\ --extra-args "--build-arg coda_version=${CODA_DEB_VERSION} --build-arg deb_repo=${CODA_DEB_REPO}" -if [[ $CODA_BUILD_ROSETTA ]]; then +if [[ -n $CODA_BUILD_ROSETTA ]]; then echo "--- Build/Release coda-rosetta to docker hub" scripts/release-docker.sh --service "coda-rosetta" --version "${CODA_VERSION}"\ --extra-args "--build-arg DUNE_PROFILE=dev --build-arg CODA_BRANCH=${CODA_GIT_BRANCH}" From 082ab4c106402332bbf76891b74fb82151a11082 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 04:50:57 +0000 Subject: [PATCH 19/33] rosetta/test-build-rosetta-dockerfile-in-ci Dont escape buildkite_branch in rebuild-deb --- scripts/rebuild-deb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index f85d7815c0c..d7a82ea1af8 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -19,7 +19,7 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -[[ -z "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g' ) +[[ -z "$BUILDKITE_BRANCH" ]] && GITBRANCH="$BUILDKITE_BRANCH" if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" From 3581d38f92127a868465bf491a45849db117c086 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 05:15:26 +0000 Subject: [PATCH 20/33] rosetta/test-build-rosetta-dockerfile-in-ci Try out a quicker iteration cycle by uploading DOCKER_DEPLOY_ENV --- buildkite/DOCKER_DEPLOY_ENV | 20 ++++++++++++++++---- buildkite/scripts/publish-deb.sh | 2 +- scripts/rebuild-deb.sh | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV index 8851715f9f8..36011ec77df 100644 --- a/buildkite/DOCKER_DEPLOY_ENV +++ b/buildkite/DOCKER_DEPLOY_ENV @@ -1,5 +1,17 @@ -export CODA_SERVICE=coda-demo -export CODA_VERSION=0.0.12-beta-develop-0be3e15 -export CODA_DEB_VERSION=0.0.12-beta-develop-0be3e15 -export CODA_DEB_REPO=develop +#export CODA_SERVICE=coda-demo +#export CODA_VERSION=0.0.12-beta-develop-0be3e15 +#export CODA_DEB_VERSION=0.0.12-beta-develop-0be3e15 +#export CODA_DEB_REPO=develop +#export NOUPLOAD=1 + +export CODA_SERVICE=coda-daemon +export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci +export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 +export CODA_PROJECT=coda-testnet-postake-medium-curves +export CODA_GIT_HASH=082ab4c +export CODA_GIT_BRANCH=rosetta-test-build-rosetta-dockerfile-in-ci +export CODA_GIT_TAG=0.0.14 +export CODA_DEB_REPO=unstable +export CODA_WAS_PUBLISHED=true +export CODA_BUILD_ROSETTA=true export NOUPLOAD=1 diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index d26ac627449..603ea5a11a3 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -55,7 +55,7 @@ if [[ "$BUILDKITE_BRANCH" == "master" ]]; then else VERSION="${GITTAG}+${BUILD_NUM}-${BUILDKITE_BRANCH}-${GITHASH}-PV${PVKEYHASH}" DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!g; s!_!-!g')" - [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true + [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta* ]] && BUILD_ROSETTA=true fi set -x diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index d7a82ea1af8..c8a46c445e0 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -19,7 +19,7 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -[[ -z "$BUILDKITE_BRANCH" ]] && GITBRANCH="$BUILDKITE_BRANCH" +[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH="$BUILDKITE_BRANCH" if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" From 5ff92a9d8f25790211ce0fa7982bcd8f64d0ce1a Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 05:23:45 +0000 Subject: [PATCH 21/33] rosetta/test-build-rosetta-dockerfile-in-ci Remove force-yes due to deprecation and quote coda_version --- dockerfiles/Dockerfile-coda-daemon | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dockerfiles/Dockerfile-coda-daemon b/dockerfiles/Dockerfile-coda-daemon index 23c9eff0adf..f2cf44a35a7 100644 --- a/dockerfiles/Dockerfile-coda-daemon +++ b/dockerfiles/Dockerfile-coda-daemon @@ -2,12 +2,14 @@ FROM ubuntu:18.04 # Run with `docker build --build-arg coda_version=` ARG coda_version ARG deb_repo=stable + +ENV DEBIAN_FRONTEND noninteractive RUN echo "Building image with version $coda_version" # Dependencies RUN apt-get -y update && \ - DEBIAN_FRONTEND=noninteractive apt-get -y upgrade && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install \ + apt-get -y upgrade && \ + apt-get -y install \ strace \ dumb-init \ libssl1.1 \ @@ -24,7 +26,7 @@ RUN apt-get -y update && \ # coda package RUN echo "deb [trusted=yes] http://packages.o1test.net $deb_repo main" > /etc/apt/sources.list.d/coda.list \ && apt-get update \ - && apt-get install --force-yes coda-testnet-postake-medium-curves=$coda_version -y + && apt-get install -y "coda-testnet-postake-medium-curves=$coda_version" WORKDIR /root From dc107f7ddc46f9630ed5effb26c689907ca70cb7 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 05:26:21 +0000 Subject: [PATCH 22/33] rosetta/test-build-rosetta-dockerfile-in-ci Try DOCKER_DEPLOY_ENV again with quoted dockerfile --- buildkite/DOCKER_DEPLOY_ENV | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV index 36011ec77df..f4969dcdee5 100644 --- a/buildkite/DOCKER_DEPLOY_ENV +++ b/buildkite/DOCKER_DEPLOY_ENV @@ -6,7 +6,7 @@ export CODA_SERVICE=coda-daemon export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci -export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 +export CODA_DEB_VERSION=0.0.14+-rosetta/test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 export CODA_PROJECT=coda-testnet-postake-medium-curves export CODA_GIT_HASH=082ab4c export CODA_GIT_BRANCH=rosetta-test-build-rosetta-dockerfile-in-ci From df65d9077d8742ade38c4366ede5cd46f108c426 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 05:31:14 +0000 Subject: [PATCH 23/33] rosetta/test-build-rosetta-dockerfile-in-ci Sanitize BUILDKITE_BRANCH for deb package name --- buildkite/DOCKER_DEPLOY_ENV | 2 +- buildkite/scripts/publish-deb.sh | 8 +++++--- scripts/rebuild-deb.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV index f4969dcdee5..36011ec77df 100644 --- a/buildkite/DOCKER_DEPLOY_ENV +++ b/buildkite/DOCKER_DEPLOY_ENV @@ -6,7 +6,7 @@ export CODA_SERVICE=coda-daemon export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci -export CODA_DEB_VERSION=0.0.14+-rosetta/test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 +export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 export CODA_PROJECT=coda-testnet-postake-medium-curves export CODA_GIT_HASH=082ab4c export CODA_GIT_BRANCH=rosetta-test-build-rosetta-dockerfile-in-ci diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 603ea5a11a3..808a3afd7ae 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -48,14 +48,16 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} +[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g') + if [[ "$BUILDKITE_BRANCH" == "master" ]]; then VERSION="${GITTAG}-${GITHASH}" DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" BUILD_ROSETTA=true else - VERSION="${GITTAG}+${BUILD_NUM}-${BUILDKITE_BRANCH}-${GITHASH}-PV${PVKEYHASH}" - DOCKER_TAG="$(echo "${GITTAG}-${BUILDKITE_BRANCH}" | sed 's!/!-!g; s!_!-!g')" - [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta* ]] && BUILD_ROSETTA=true + VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" + DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!g; s!_!-!g')" + [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true fi set -x diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index c8a46c445e0..43267b51441 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -19,7 +19,7 @@ PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" BUILD_NUM=${BUILDKITE_BUILD_NUM} BUILD_URL=${BUILDKITE_BUILD_URL} -[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH="$BUILDKITE_BRANCH" +[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g') if [ "$GITBRANCH" == "master" ]; then VERSION="${GITTAG}-${GITHASH}" From a00ddf70d1714fed47df54cc3a7a03a97bdc06f4 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 05:49:14 +0000 Subject: [PATCH 24/33] rosetta/test-build-rosetta-dockerfile-in-ci Disbale docker_deploy_env quick test --- buildkite/DOCKER_DEPLOY_ENV | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 buildkite/DOCKER_DEPLOY_ENV diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV deleted file mode 100644 index 36011ec77df..00000000000 --- a/buildkite/DOCKER_DEPLOY_ENV +++ /dev/null @@ -1,17 +0,0 @@ -#export CODA_SERVICE=coda-demo -#export CODA_VERSION=0.0.12-beta-develop-0be3e15 -#export CODA_DEB_VERSION=0.0.12-beta-develop-0be3e15 -#export CODA_DEB_REPO=develop -#export NOUPLOAD=1 - -export CODA_SERVICE=coda-daemon -export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci -export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-082ab4c-PVe8e43745 -export CODA_PROJECT=coda-testnet-postake-medium-curves -export CODA_GIT_HASH=082ab4c -export CODA_GIT_BRANCH=rosetta-test-build-rosetta-dockerfile-in-ci -export CODA_GIT_TAG=0.0.14 -export CODA_DEB_REPO=unstable -export CODA_WAS_PUBLISHED=true -export CODA_BUILD_ROSETTA=true -export NOUPLOAD=1 From 2b87dad427e6b10d0350c9938e107b83fc485849 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 06:09:41 +0000 Subject: [PATCH 25/33] rosetta/test-build-rosetta-dockerfile-in-ci Make sure rosetta=true works while also trying to fix the env in publish-deb --- buildkite/DOCKER_DEPLOY_ENV | 10 ++++++++++ buildkite/scripts/publish-deb.sh | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 buildkite/DOCKER_DEPLOY_ENV diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV new file mode 100644 index 00000000000..778cdd40d8b --- /dev/null +++ b/buildkite/DOCKER_DEPLOY_ENV @@ -0,0 +1,10 @@ +export CODA_SERVICE=coda-daemon +export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci +export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-a00ddf7-PVe8e43745 +export CODA_PROJECT=coda-testnet-postake-medium-curves +export CODA_GIT_HASH=a00ddf7 +export CODA_GIT_BRANCH=rosetta/test-build-rosetta-dockerfile-in-ci +export CODA_GIT_TAG=0.0.14 +export CODA_DEB_REPO=unstable +export CODA_WAS_PUBLISHED=true +export CODA_BUILD_ROSETTA=true diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 808a3afd7ae..75317731cfd 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -35,7 +35,7 @@ set +x echo "Exporting Variables: " GITHASH=$(git rev-parse --short=7 HEAD) -#GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) +GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) GITTAG=$(git describe --abbrev=0) @@ -57,7 +57,7 @@ if [[ "$BUILDKITE_BRANCH" == "master" ]]; then else VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!g; s!_!-!g')" - [[ "${BUILDKITE_BRANCH}" == "develop" ]] || [[ "${BUILDKITE_BRANCH}" == rosetta/* ]] && BUILD_ROSETTA=true + [[ "${BUILDKITE_BRANCH}" == "develop" || "${GITBRANCH}" == rosetta-* ]] && BUILD_ROSETTA=true fi set -x From 590829a0ebc1033f845cbabe37179a8a54a712b6 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 06:14:56 +0000 Subject: [PATCH 26/33] rosetta/test-build-rosetta-dockerfile-in-ci Use a case statement building rosetta --- buildkite/scripts/publish-deb.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 75317731cfd..623779c89cb 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -53,13 +53,15 @@ BUILD_URL=${BUILDKITE_BUILD_URL} if [[ "$BUILDKITE_BRANCH" == "master" ]]; then VERSION="${GITTAG}-${GITHASH}" DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" - BUILD_ROSETTA=true else VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!g; s!_!-!g')" - [[ "${BUILDKITE_BRANCH}" == "develop" || "${GITBRANCH}" == rosetta-* ]] && BUILD_ROSETTA=true fi +case $BUILDKITE_BRANCH in master|develop|rosetta*) + BUILD_ROSETTA=true +esac + set -x # Export variables for use with downstream steps echo "export CODA_SERVICE=coda-daemon" >> ./DOCKER_DEPLOY_ENV @@ -71,6 +73,6 @@ echo "export CODA_GIT_BRANCH=${BUILDKITE_BRANCH}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_BUILD_ROSETTA=$ROSETTA" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_BUILD_ROSETTA=$BUILD_ROSETTA" >> ./DOCKER_DEPLOY_ENV set +x From cdd9110c23ac87d22b96acc8973581bd537e2d2f Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 06:16:03 +0000 Subject: [PATCH 27/33] rosetta/test-build-rosetta-dockerfile-in-ci Go back to the DEPLOY_ENV from master --- buildkite/DOCKER_DEPLOY_ENV | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/buildkite/DOCKER_DEPLOY_ENV b/buildkite/DOCKER_DEPLOY_ENV index 778cdd40d8b..8851715f9f8 100644 --- a/buildkite/DOCKER_DEPLOY_ENV +++ b/buildkite/DOCKER_DEPLOY_ENV @@ -1,10 +1,5 @@ -export CODA_SERVICE=coda-daemon -export CODA_VERSION=0.0.14-rosetta-test-build-rosetta-dockerfile-in-ci -export CODA_DEB_VERSION=0.0.14+-rosetta-test-build-rosetta-dockerfile-in-ci-a00ddf7-PVe8e43745 -export CODA_PROJECT=coda-testnet-postake-medium-curves -export CODA_GIT_HASH=a00ddf7 -export CODA_GIT_BRANCH=rosetta/test-build-rosetta-dockerfile-in-ci -export CODA_GIT_TAG=0.0.14 -export CODA_DEB_REPO=unstable -export CODA_WAS_PUBLISHED=true -export CODA_BUILD_ROSETTA=true +export CODA_SERVICE=coda-demo +export CODA_VERSION=0.0.12-beta-develop-0be3e15 +export CODA_DEB_VERSION=0.0.12-beta-develop-0be3e15 +export CODA_DEB_REPO=develop +export NOUPLOAD=1 From 78d5a499b7fff885b50cd40150949517e5b89a6f Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 16:57:36 +0000 Subject: [PATCH 28/33] rosetta/test-build-rosetta-dockerfile-in-ci Centralize git env in a script --- buildkite/scripts/export-git-env-vars.sh | 37 ++++++++++++++++++++++++ buildkite/scripts/publish-deb.sh | 29 ++----------------- scripts/rebuild-deb.sh | 27 +++-------------- 3 files changed, 43 insertions(+), 50 deletions(-) create mode 100755 buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh new file mode 100755 index 00000000000..a80f9e474bf --- /dev/null +++ b/buildkite/scripts/export-git-env-vars.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -eo pipefail + +set +x +echo "Exporting Variables: " + +export GITHASH=$(git rev-parse --short=7 HEAD) +export GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) +export GITTAG=$(git describe --abbrev=0) + + +# Identify All Artifacts by Branch and Git Hash +set +u +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" + +export PVKEYHASH=$(${SCRIPTPATH}/../../_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) + +export PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" + +export BUILD_NUM=${BUILDKITE_BUILD_NUM} +export BUILD_URL=${BUILDKITE_BUILD_URL} + +[[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g') + +if [[ "$BUILDKITE_BRANCH" == "master" ]]; then + export VERSION="${GITTAG}-${GITHASH}" + export DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" +else + export VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" + export DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!g; s!_!-!g')" +fi + +case $BUILDKITE_BRANCH in master|develop|rosetta*) + export BUILD_ROSETTA=true +esac + +set -x diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index 623779c89cb..f12426f5aad 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -34,33 +34,8 @@ ${DEBS3} --codename "${CODENAME}" "${DEBS}" set +x echo "Exporting Variables: " -GITHASH=$(git rev-parse --short=7 HEAD) -GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) -GITTAG=$(git describe --abbrev=0) - - -# Identify All Artifacts by Branch and Git Hash -set +u -PVKEYHASH=$(./_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) - -PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" - -BUILD_NUM=${BUILDKITE_BUILD_NUM} -BUILD_URL=${BUILDKITE_BUILD_URL} - -[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g') - -if [[ "$BUILDKITE_BRANCH" == "master" ]]; then - VERSION="${GITTAG}-${GITHASH}" - DOCKER_TAG="$(echo "${VERSION}" | sed 's!/!-!; s!_!-!g')" -else - VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" - DOCKER_TAG="$(echo "${GITTAG}-${GITBRANCH}" | sed 's!/!-!g; s!_!-!g')" -fi - -case $BUILDKITE_BRANCH in master|develop|rosetta*) - BUILD_ROSETTA=true -esac +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +source "${SCRIPTPATH}/export-git-env-vars.sh" set -x # Export variables for use with downstream steps diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 43267b51441..ed403034522 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -5,34 +5,15 @@ set -euo pipefail SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -cd "${SCRIPTPATH}/../_build" -GITHASH=$(git rev-parse --short=7 HEAD) -GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD | sed 's!/!-!; s!_!-!g' ) -GITTAG=$(git describe --abbrev=0) +# Load in env vars for githash/branch/etc. +source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" + +cd "${SCRIPTPATH}/../_build" -# Identify All Artifacts by Branch and Git Hash set +u PVKEYHASH=$(./default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) -PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" - -BUILD_NUM=${BUILDKITE_BUILD_NUM} -BUILD_URL=${BUILDKITE_BUILD_URL} -[[ -n "$BUILDKITE_BRANCH" ]] && GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!; s!_!-!g') - -if [ "$GITBRANCH" == "master" ]; then - VERSION="${GITTAG}-${GITHASH}" -else - VERSION="${GITTAG}+${BUILD_NUM}-${GITBRANCH}-${GITHASH}-PV${PVKEYHASH}" -fi - -# Export variables for use with downstream steps -echo "export CODA_DEB_VERSION=$VERSION" >> /tmp/DOCKER_DEPLOY_ENV -echo "export CODA_PROJECT=$PROJECT" >> /tmp/DOCKER_DEPLOY_ENV -echo "export CODA_GIT_HASH=$GITHASH" >> /tmp/DOCKER_DEPLOY_ENV -echo "export CODA_GIT_BRANCH=$GITBRANCH" >> /tmp/DOCKER_DEPLOY_ENV -echo "export CODA_GIT_TAG=$GITTAG" >> /tmp/DOCKER_DEPLOY_ENV BUILDDIR="deb_build" From b3dd02af518902f1bf3693abcc1cfb04ce000ec3 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 16:59:22 +0000 Subject: [PATCH 29/33] rosetta/test-build-rosetta-dockerfile-in-ci Improve consistency in syntax --- buildkite/scripts/publish-deb.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index f12426f5aad..ea84e99ad01 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -42,12 +42,12 @@ set -x echo "export CODA_SERVICE=coda-daemon" >> ./DOCKER_DEPLOY_ENV echo "export CODA_VERSION=${DOCKER_TAG}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_DEB_VERSION=${VERSION}" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_PROJECT=$PROJECT" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_GIT_HASH=$GITHASH" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_PROJECT=${PROJECT}" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_HASH=${GITHASH}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_GIT_BRANCH=${BUILDKITE_BRANCH}" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_GIT_TAG=$GITTAG" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_DEB_REPO=$CODENAME" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_GIT_TAG=${GITTAG}" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_DEB_REPO=${CODENAME}" >> ./DOCKER_DEPLOY_ENV echo "export CODA_WAS_PUBLISHED=true" >> ./DOCKER_DEPLOY_ENV -echo "export CODA_BUILD_ROSETTA=$BUILD_ROSETTA" >> ./DOCKER_DEPLOY_ENV +echo "export CODA_BUILD_ROSETTA=${BUILD_ROSETTA}" >> ./DOCKER_DEPLOY_ENV set +x From 64e7108fd1f8f60d6424b88dadd0f75f6790369f Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 17:27:49 +0000 Subject: [PATCH 30/33] rosetta/test-build-rosetta-dockerfile-in-ci Dont calculate pvhash twice, hope that its consistent --- scripts/rebuild-deb.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index ed403034522..471a90c8902 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -11,10 +11,6 @@ source "${SCRIPTPATH}/../buildkite/scripts/export-git-env-vars.sh" cd "${SCRIPTPATH}/../_build" -set +u -PVKEYHASH=$(./default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) - - BUILDDIR="deb_build" mkdir -p "${BUILDDIR}/DEBIAN" From 8507462907de976b376b701d4ce18913241f11b7 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 18:34:24 +0000 Subject: [PATCH 31/33] rosetta/test-build-rosetta-dockerfile-in-ci Fix path to coda in export-git-env-vars --- buildkite/scripts/export-git-env-vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index a80f9e474bf..a02c8dcd4c5 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -13,7 +13,7 @@ export GITTAG=$(git describe --abbrev=0) set +u SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -export PVKEYHASH=$(${SCRIPTPATH}/../../_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) +export PVKEYHASH=$(${SCRIPTPATH}/../_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) export PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" From 556e087f95b1120feb0364ef2cecdfb4606f34a0 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 19:10:43 +0000 Subject: [PATCH 32/33] rosetta/test-build-rosetta-dockerfile-in-ci Hardcode buildkite path into export-git-env-vars --- buildkite/scripts/export-git-env-vars.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/scripts/export-git-env-vars.sh b/buildkite/scripts/export-git-env-vars.sh index a02c8dcd4c5..acb7b3251fc 100755 --- a/buildkite/scripts/export-git-env-vars.sh +++ b/buildkite/scripts/export-git-env-vars.sh @@ -11,9 +11,8 @@ export GITTAG=$(git describe --abbrev=0) # Identify All Artifacts by Branch and Git Hash set +u -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -export PVKEYHASH=$(${SCRIPTPATH}/../_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) +export PVKEYHASH=$(/workdir/_build/default/src/app/cli/src/coda.exe internal snark-hashes | sort | md5sum | cut -c1-8) export PROJECT="coda-$(echo "$DUNE_PROFILE" | tr _ -)" From 467b6b469834912b570e81b2777eae7325e1e2f1 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 13 Aug 2020 20:02:55 +0000 Subject: [PATCH 33/33] rosetta/test-build-rosetta-dockerfile-in-ci Add back /tmp/DOCKER_DEPLOY_ENV setup to avoid breaking circle-ci, with comment about current use --- scripts/rebuild-deb.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh index 471a90c8902..147050e8b60 100755 --- a/scripts/rebuild-deb.sh +++ b/scripts/rebuild-deb.sh @@ -149,3 +149,11 @@ rm -f "${BUILDDIR}"/var/lib/coda/*_proving # build another deb fakeroot dpkg-deb --build "${BUILDDIR}" ${PROJECT}-noprovingkeys_${VERSION}.deb ls -lh coda*.deb + + +# Export variables for use with downstream circle-ci steps (see buildkite/scripts/publish-deb.sh for BK DOCKER_DEPLOY_ENV) +echo "export CODA_DEB_VERSION=$VERSION" >> /tmp/DOCKER_DEPLOY_ENV +echo "export CODA_PROJECT=$PROJECT" >> /tmp/DOCKER_DEPLOY_ENV +echo "export CODA_GIT_HASH=$GITHASH" >> /tmp/DOCKER_DEPLOY_ENV +echo "export CODA_GIT_BRANCH=$GITBRANCH" >> /tmp/DOCKER_DEPLOY_ENV +echo "export CODA_GIT_TAG=$GITTAG" >> /tmp/DOCKER_DEPLOY_ENV