From ed956ebd6a01fe830d00958ba99138c136295634 Mon Sep 17 00:00:00 2001 From: Micah Abbott Date: Thu, 20 Sep 2018 11:48:37 -0400 Subject: [PATCH 1/4] treecompose: change tagging scheme for container image In an effort to get closer to what has been discussed in openshift/os#150, the tagging scheme has been changed so that the latest container image out of the pipeline is tagged with `buildmaster` and the commit ID. The usage of the commit ID allows for other parts of the piepline to easily refer to it for other operations. --- Jenkinsfile.treecompose | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile.treecompose b/Jenkinsfile.treecompose index ad615627..680000d9 100644 --- a/Jenkinsfile.treecompose +++ b/Jenkinsfile.treecompose @@ -2,13 +2,11 @@ def TIMER = "H/30 * * * *" def NODE = "rhcos-jenkins" def API_CI_REGISTRY = "registry.svc.ci.openshift.org" def OS_NAME = "maipo"; -def OSCONTAINER_IMG = API_CI_REGISTRY + "/rhcos/os-${OS_NAME}:latest" +def OSCONTAINER_IMG = API_CI_REGISTRY + "/rhcos/os-${OS_NAME}" def COMPOSEFLAGS = ""; // We write to this one for now def artifact_repo = "/srv/rhcos/output/repo" -// We write pkg_diff.txt here -def images = "/srv/rhcos/output/images" node(NODE) { def par_stages = [:] @@ -90,8 +88,9 @@ node(NODE) { stage("Build new container") { sh """ podman build --build-arg OS_VERSION=${composeMeta.version} \ --build-arg OS_COMMIT=${composeMeta.commit} \ - -t ${OSCONTAINER_IMG} \ + -t ${OSCONTAINER_IMG}:buildmaster \ -f ${WORKSPACE}/Dockerfile.rollup ${WORKSPACE} + podman tag ${OSCONTAINER_IMG}:buildmaster ${OSCONTAINER_IMG}:${composeMeta.commit} """ } if (params.DRY_RUN) { @@ -102,8 +101,9 @@ node(NODE) { } stage("Push container") { sh """ - podman push ${OSCONTAINER_IMG} - skopeo inspect docker://${OSCONTAINER_IMG} | jq '.Digest' > imgid.txt + podman push ${OSCONTAINER_IMG}:buildmaster + podman push ${OSCONTAINER_IMG}:${composeMeta.commit} + skopeo inspect docker://${OSCONTAINER_IMG}:buildmaster | jq '.Digest' > imgid.txt """ def cid = readFile('imgid.txt').trim().replaceAll('"',''); currentBuild.description = "🆕 ${OSCONTAINER_IMG}@${cid} (${composeMeta.version})"; @@ -119,7 +119,7 @@ node(NODE) { --dest ${ARTIFACT_SERVER}:${artifact_repo} --src=${repo}/ \ --rsync-opt=--stats --rsync-opt=-e \ --rsync-opt='ssh -i ${KEY_FILE} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' - """ + """ } } stage("Cleanup") { sh """ From e30a4edfc54a58c5b46ac0907c204749f25e55ff Mon Sep 17 00:00:00 2001 From: Micah Abbott Date: Thu, 20 Sep 2018 11:54:03 -0400 Subject: [PATCH 2/4] aws-test: tag and push an `alpha` of the oscontainer In openshift/os#150, it was discussed that an `alpha` tag should be made for the oscontainer (and cloud image) after it passes the tests run in AWS. This change accomplishes this goal by pulling the oscontainer by commit ID, tagging it as `alpha` and pushing it to the registry. (After a successful test, obviously) If the AWS tests fail, the image tagged with the ostree commit is garbage collected and no alpha promotion happens. --- Jenkinsfile.aws-test | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile.aws-test b/Jenkinsfile.aws-test index 9e03678f..32fe41e0 100644 --- a/Jenkinsfile.aws-test +++ b/Jenkinsfile.aws-test @@ -1,5 +1,8 @@ def NODE = "rhcos-jenkins" def AWS_REGION = "us-east-1" +def API_CI_REGISTRY = "registry.svc.ci.openshift.org" +def OS_NAME = "maipo"; +def OSCONTAINER_IMG = API_CI_REGISTRY + "/rhcos/os-${OS_NAME}" // location on the server we'll rsync to/from our $WORKSPACE def images = "/srv/rhcos/output/images" @@ -23,7 +26,7 @@ node(NODE) { // We're only ever triggered by the cloud job, so we know the latest build is in latest/ // We immediately resolve it back to the specific images/ dir - def version + def ostree_commit, version try { utils.inside_assembler_container("") { stage("Sync In") { @@ -32,6 +35,7 @@ node(NODE) { sshUserPrivateKey(credentialsId: params.ARTIFACT_SSH_CREDS_ID, keyFileVariable: 'KEY_FILE'), ]) { utils.rsync_file_in_dest(ARTIFACT_SERVER, KEY_FILE, "${images}/cloud/latest/meta.json", "${WORKSPACE}/meta.json") + ostree_commit = utils.sh_capture("jq -r '.[\"ostree-commit\"]' ${WORKSPACE}/meta.json") version = utils.sh_capture("jq -r '.[\"ostree-version\"]' ${WORKSPACE}/meta.json") // resolve to original dir to avoid races in the next rsync in def imgv = utils.sh_capture("jq -r '.[\"image-version\"]' ${WORKSPACE}/meta.json") @@ -48,12 +52,21 @@ node(NODE) { string(credentialsId: params.S3_PRIVATE_BUCKET, variable: 'S3_PRIVATE_BUCKET'), string(credentialsId: params.AWS_CI_ACCOUNT, variable: 'AWS_CI_ACCOUNT'), string(credentialsId: params.S3_PUBLIC_BUCKET, variable: 'S3_PUBLIC_BUCKET'), + usernameColonPassword(credentialsId: params.REGISTRY_CREDENTIALS, variable: 'CREDS'), ]) { def ami_intermediate = utils.sh_capture("jq -r .HVM ${WORKSPACE}/aws-${AWS_REGION}.json") + + // login to registry and setup container storage + def (registryUser, registryPass) = "${CREDS}".split(':') + utils.registry_login(registryUser, registryPass, "${API_CI_REGISTRY}") + utils.prep_container_storage("${WORKSPACE}") + currentBuild.description = "version=${version} ami=${ami_intermediate}" sh """ # Do testing with intermediate aws image passed in by cloud job if ! kola -b rhcos -p aws --aws-type t2.small --tapfile rhcos-aws.tap --aws-ami ${ami_intermediate} --aws-region ${AWS_REGION} -j ${NUM_VMS} run; then + # if the tests fail, GC the container image tagged with the ostree commit + skopeo delete docker://${OSCONTAINER_IMG}:${ostree_commit} exit 1 fi @@ -71,6 +84,10 @@ node(NODE) { aws s3 cp --acl public-read \ ${WORKSPACE}/aws-${AWS_REGION}.json \ s3://${S3_PUBLIC_BUCKET}/aws-${AWS_REGION}-tested.json + + # Copy the container image to alpha, then GC the image tagged with the ostree commit + skopeo copy docker://${OSCONTAINER_IMG}:${ostree_commit} docker://${OSCONTAINER_IMG}:alpha + skopeo delete docker://${OSCONTAINER_IMG}:${ostree_commit} """ } } From ede4d520546520a58540b5b1919b6b33da211d1c Mon Sep 17 00:00:00 2001 From: Micah Abbott Date: Thu, 27 Sep 2018 11:09:02 -0400 Subject: [PATCH 3/4] treecompose: detect if `buildmaster` exists on registry To bootstrap the change to `buildmaster`, we need to do a quick test if the tag exists on the registry. If it exists, we can pull from it, otherwise we need to use `latest` --- Jenkinsfile.treecompose | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile.treecompose b/Jenkinsfile.treecompose index 680000d9..a6293797 100644 --- a/Jenkinsfile.treecompose +++ b/Jenkinsfile.treecompose @@ -36,7 +36,15 @@ node(NODE) { withCredentials([ usernameColonPassword(credentialsId: params.REGISTRY_CREDENTIALS, variable: 'CREDS'), ]) { - sh """./scripts/pull-mount-oscontainer ${API_CI_REGISTRY} ${treecompose_workdir} ${OSCONTAINER_IMG}""" + def (registryUser, registryPass) = "${CREDS}".split(':') + utils.registry_login(registryUser, registryPass, "${API_CI_REGISTRY}") + sh """ + if ! skopeo inspect docker://${OSCONTAINER_IMG}:buildmaster; then + ./scripts/pull-mount-oscontainer ${API_CI_REGISTRY} ${treecompose_workdir} ${OSCONTAINER_IMG}:latest + else + ./scripts/pull-mount-oscontainer ${API_CI_REGISTRY} ${treecompose_workdir} ${OSCONTAINER_IMG}:buildmaster + fi + """ } } From e4f784a5d405824b6149a34489ad3095c54d580d Mon Sep 17 00:00:00 2001 From: Micah Abbott Date: Fri, 28 Sep 2018 17:26:42 -0400 Subject: [PATCH 4/4] aws-test: use readJSON to extract values from JSON files --- Jenkinsfile.aws-test | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile.aws-test b/Jenkinsfile.aws-test index 32fe41e0..ea754864 100644 --- a/Jenkinsfile.aws-test +++ b/Jenkinsfile.aws-test @@ -26,7 +26,7 @@ node(NODE) { // We're only ever triggered by the cloud job, so we know the latest build is in latest/ // We immediately resolve it back to the specific images/ dir - def ostree_commit, version + def latest_meta, ostree_commit, version try { utils.inside_assembler_container("") { stage("Sync In") { @@ -35,16 +35,18 @@ node(NODE) { sshUserPrivateKey(credentialsId: params.ARTIFACT_SSH_CREDS_ID, keyFileVariable: 'KEY_FILE'), ]) { utils.rsync_file_in_dest(ARTIFACT_SERVER, KEY_FILE, "${images}/cloud/latest/meta.json", "${WORKSPACE}/meta.json") - ostree_commit = utils.sh_capture("jq -r '.[\"ostree-commit\"]' ${WORKSPACE}/meta.json") - version = utils.sh_capture("jq -r '.[\"ostree-version\"]' ${WORKSPACE}/meta.json") + latest_meta = readJSON file: "meta.json"; + ostree_commit = latest_meta["ostree-commit"]; + version = latest_meta["ostree-version"]; // resolve to original dir to avoid races in the next rsync in - def imgv = utils.sh_capture("jq -r '.[\"image-version\"]' ${WORKSPACE}/meta.json") + def imgv = latest_meta["image-version"] utils.rsync_file_in_dest(ARTIFACT_SERVER, KEY_FILE, "${images}/cloud/${imgv}/aws-${AWS_REGION}.json", "${WORKSPACE}/aws-${AWS_REGION}.json") } } // Number of parallel kola tests def NUM_VMS = "10" + def aws_json try { stage("Run Kola tests on intermediate aws image") { withCredentials([ @@ -54,7 +56,8 @@ node(NODE) { string(credentialsId: params.S3_PUBLIC_BUCKET, variable: 'S3_PUBLIC_BUCKET'), usernameColonPassword(credentialsId: params.REGISTRY_CREDENTIALS, variable: 'CREDS'), ]) { - def ami_intermediate = utils.sh_capture("jq -r .HVM ${WORKSPACE}/aws-${AWS_REGION}.json") + aws_json = readJSON file: "aws-${AWS_REGION}.json"; + def ami_intermediate = aws_json["HVM"]; // login to registry and setup container storage def (registryUser, registryPass) = "${CREDS}".split(':') @@ -95,6 +98,7 @@ node(NODE) { sh 'if test -e _kola_temp; then tar -cJf _kola_temp.tar.xz _kola_temp; fi' archiveArtifacts artifacts: "_kola_temp.tar.xz", allowEmptyArchive: true archiveArtifacts artifacts: "rhcos-aws.tap", allowEmptyArchive: true + archiveArtifacts artifacts: "*.json", allowEmptyArchive: true } stage("rsync out") {