From 3cd405d090ba9fa80ba5a2cc1d79922875ff6ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 12:51:40 +0200 Subject: [PATCH 1/9] feat: push aliases for docker images --- .ci/packaging.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 4145ee6bdd13..1f4865d7e98a 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -193,8 +193,12 @@ def pushCIDockerImages(){ def tagAndPush(name){ def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() + def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ + aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version + libbetaVer += "-SNAPSHOT" + aliasVersion += "-SNAPSHOT" } def tagName = "${libbetaVer}" @@ -227,6 +231,24 @@ def tagAndPush(name){ log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") } } + + if (aliasVersion != "") { + def aliasName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${aliasVersion}" + def iterations = 0 + retryWithSleep(retries: 3, seconds: 5, backoff: true) { + iterations++ + def status = sh(label:'Change tag and push', script: """ + docker tag ${oldName} ${aliasName} + docker push ${newName} + """, returnStatus: true) + + if ( status > 0 && iterations < 3) { + error('tag and push failed, retry') + } else if ( status > 0 ) { + log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") + } + } + } } } From 707e0d71556553b15388adec0c7118ff89210ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 13:19:22 +0200 Subject: [PATCH 2/9] feat: build alias for snapshots --- dev-tools/mage/dockerbuilder.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 90a994348846..4439f9cc75ed 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -182,13 +182,26 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in func (b *dockerBuilder) dockerBuild() (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) + aliasTag := "" if b.Snapshot { tag = tag + "-SNAPSHOT" + // remove third number in version + aliasTag = fmt.Sprintf("%s:%s-SNAPSHOT", b.imageName, b.Version[0:strings.LastIndex(b.Version, ".")]) } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) + aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) } - return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) + + args := []string{ + "build", "-t", tag, + } + if aliasTag != "" { + args = append(args, "-t", aliasTag) + } + args = append(args, b.buildDir) + + return tag, sh.Run("docker", args...) } func (b *dockerBuilder) dockerSave(tag string) error { From cff2cef82cb107bfddeca5caf225a9307db72135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 16:04:22 +0200 Subject: [PATCH 3/9] fix: only update alias on snapshots Co-authored-by: Jaime Soriano Pastor --- dev-tools/mage/dockerbuilder.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 4439f9cc75ed..89d1a926f6ed 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -190,7 +190,9 @@ func (b *dockerBuilder) dockerBuild() (string, error) { } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) - aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) + if b.Snapshot { + aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) + } } args := []string{ From 843f472bde35563af11fea6023a684ac9da0c773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 16:14:23 +0200 Subject: [PATCH 4/9] fix: wrong image name for alias --- .ci/packaging.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 1f4865d7e98a..5d192b243b6f 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -239,7 +239,7 @@ def tagAndPush(name){ iterations++ def status = sh(label:'Change tag and push', script: """ docker tag ${oldName} ${aliasName} - docker push ${newName} + docker push ${aliasName} """, returnStatus: true) if ( status > 0 && iterations < 3) { From 64949ffd7c9dd148641173e06c529512b6c93dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 16:15:26 +0200 Subject: [PATCH 5/9] fix: reuse variable as groovy does not hide variables by scope --- .ci/packaging.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 5d192b243b6f..a73aa8a4714f 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -234,7 +234,7 @@ def tagAndPush(name){ if (aliasVersion != "") { def aliasName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${aliasVersion}" - def iterations = 0 + iterations = 0 retryWithSleep(retries: 3, seconds: 5, backoff: true) { iterations++ def status = sh(label:'Change tag and push', script: """ From d2a3f393e2beaebb562e5535953ef94f5c73b2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 17:35:43 +0200 Subject: [PATCH 6/9] chore: extract common logic to a method --- .ci/packaging.groovy | 62 ++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index a73aa8a4714f..9d0c82f25292 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -191,7 +191,7 @@ def pushCIDockerImages(){ } } -def tagAndPush(name){ +def tagAndPush(beatName){ def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim() def aliasVersion = "" if("${env.SNAPSHOT}" == "true"){ @@ -211,43 +211,37 @@ def tagAndPush(name){ // supported image flavours def variants = ["", "-oss", "-ubi8"] variants.each { variant -> - def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}" - def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}" - def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}" + doTagAndPush(beatName, variant, libbetaVer, tagName) + doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - def iterations = 0 - retryWithSleep(retries: 3, seconds: 5, backoff: true) { - iterations++ - def status = sh(label:'Change tag and push', script: """ - docker tag ${oldName} ${newName} - docker push ${newName} - docker tag ${oldName} ${commitName} - docker push ${commitName} - """, returnStatus: true) - - if ( status > 0 && iterations < 3) { - error('tag and push failed, retry') - } else if ( status > 0 ) { - log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") - } + if (aliasVersion != "") { + doTagAndPush(beatName, variant, libbetaVer, aliasVersion) } + } +} - if (aliasVersion != "") { - def aliasName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${aliasVersion}" - iterations = 0 - retryWithSleep(retries: 3, seconds: 5, backoff: true) { - iterations++ - def status = sh(label:'Change tag and push', script: """ - docker tag ${oldName} ${aliasName} - docker push ${aliasName} - """, returnStatus: true) +/** +* @param beatName name of the Beat +* @param variant name of the variant used to build the docker image name +* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace +* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace +*/ +def doTagAndPush(beatName, variant, sourceTag, targetTag) { + def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}" + def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}" - if ( status > 0 && iterations < 3) { - error('tag and push failed, retry') - } else if ( status > 0 ) { - log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") - } - } + def iterations = 0 + retryWithSleep(retries: 3, seconds: 5, backoff: true) { + iterations++ + def status = sh(label: "Change tag and push ${targetName}", script: """ + docker tag ${sourceName} ${targetName} + docker push ${targetName} + """, returnStatus: true) + + if ( status > 0 && iterations < 3) { + error("tag and push failed for ${beatName}, retry") + } else if ( status > 0 ) { + log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") } } } From 4724960d545ee0b6caeb28581d4d0ef4a947f882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 17:36:44 +0200 Subject: [PATCH 7/9] Revert "fix: only update alias on snapshots" This reverts commit cff2cef82cb107bfddeca5caf225a9307db72135. --- dev-tools/mage/dockerbuilder.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 89d1a926f6ed..4439f9cc75ed 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -190,9 +190,7 @@ func (b *dockerBuilder) dockerBuild() (string, error) { } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) - if b.Snapshot { - aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) - } + aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) } args := []string{ From 7416af7c3d704cc27fa50f1c921572b7053f3058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 17:36:51 +0200 Subject: [PATCH 8/9] Revert "feat: build alias for snapshots" This reverts commit 707e0d71556553b15388adec0c7118ff89210ac9. --- dev-tools/mage/dockerbuilder.go | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 4439f9cc75ed..90a994348846 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -182,26 +182,13 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in func (b *dockerBuilder) dockerBuild() (string, error) { tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) - aliasTag := "" if b.Snapshot { tag = tag + "-SNAPSHOT" - // remove third number in version - aliasTag = fmt.Sprintf("%s:%s-SNAPSHOT", b.imageName, b.Version[0:strings.LastIndex(b.Version, ".")]) } if repository, _ := b.ExtraVars["repository"]; repository != "" { tag = fmt.Sprintf("%s/%s", repository, tag) - aliasTag = fmt.Sprintf("%s/%s", repository, aliasTag) } - - args := []string{ - "build", "-t", tag, - } - if aliasTag != "" { - args = append(args, "-t", aliasTag) - } - args = append(args, b.buildDir) - - return tag, sh.Run("docker", args...) + return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) } func (b *dockerBuilder) dockerSave(tag string) error { From e8ab32b8ffaf2fdba2024164271d14074f21db7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 19 Oct 2020 18:21:45 +0200 Subject: [PATCH 9/9] chore: do not push aliases for PRs --- .ci/packaging.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 9d0c82f25292..2a67e3f7ea03 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -214,7 +214,7 @@ def tagAndPush(beatName){ doTagAndPush(beatName, variant, libbetaVer, tagName) doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}") - if (aliasVersion != "") { + if (!isPR() && aliasVersion != "") { doTagAndPush(beatName, variant, libbetaVer, aliasVersion) } }