From c7324e117dc2150c4f959fca7a763845ebbf036d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 8 Nov 2024 19:19:23 +0100 Subject: [PATCH 01/12] Remove a shared token for triggering builds Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/GenericRemoteToken.groovy | 34 ------------------- .../dsl/_configs_/OSRFLinuxBackportPkg.groovy | 2 -- .../dsl/_configs_/OSRFLinuxBuildPkg.groovy | 2 -- .../dsl/_configs_/OSRFSourceCreation.groovy | 1 - jenkins-scripts/dsl/brew_release.dsl | 4 --- jenkins-scripts/dsl/gazebo_ros_pkgs.dsl | 1 - jenkins-scripts/dsl/ros_gz_bridge.dsl | 1 - 7 files changed, 45 deletions(-) delete mode 100644 jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy diff --git a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy b/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy deleted file mode 100644 index 07e29d7c2..000000000 --- a/jenkins-scripts/dsl/_configs_/GenericRemoteToken.groovy +++ /dev/null @@ -1,34 +0,0 @@ -package _configs_ - -import javaposse.jobdsl.dsl.Job - -class GenericRemoteToken -{ - // FIXME getEnvVars can not be called in a static scope. Hardcoded by now. - // static File token_file = new File(build.getEnvVars()['HOME'] + '/remote_token') - static File token_file = new File('/var/lib/jenkins/remote_token') - - static void create(Job job) - { - if (! token_file.exists()) { - println("!!! token file was not found for setting the remote password") - println("check your filesystem in the jenkins node for: ") - println(token_file) - // We can not use exit here, DSL job hangs - job.with - { - steps { - setBuildResult('UNSTABLE') - } - } - } - - job.with - { - // remote calls don't have DSL implementation - configure { project -> - project / authToken(token_file.text.replaceAll("[\n\r]", "")) - } - } - } -} diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy index 077e57e56..d3b28888d 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBackportPkg.groovy @@ -4,7 +4,6 @@ import javaposse.jobdsl.dsl.Job /* -> OSRFLinuxBase - -> GenericRemoteToken Implements: - priorioty 300 @@ -24,7 +23,6 @@ class OSRFLinuxBackportPkg static void create(Job job) { OSRFLinuxBase.create(job) - GenericRemoteToken.create(job) job.with { diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 41d8aac15..97ad235a3 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -5,7 +5,6 @@ import _configs_.Globals /* -> OSRFLinuxBuildPkgBase - -> GenericRemoteToken Implements: - priority 100 @@ -28,7 +27,6 @@ class OSRFLinuxBuildPkg static void create(Job job, Map default_params = [:]) { OSRFLinuxBuildPkgBase.create(job) - GenericRemoteToken.create(job) job.with { diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index 2f8f490bb..827d93238 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -57,7 +57,6 @@ class OSRFSourceCreation static void create(Job job, Map default_params = [:], Map default_hidden_params = [:]) { OSRFLinuxBuildPkgBase.create(job) - GenericRemoteToken.create(job) OSRFSourceCreation.addParameters(job, default_params) def pkg_sources_dir="pkgs" diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 69f847fa0..3644b0c31 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -51,7 +51,6 @@ void include_common_params(Job job) // 1. BREW pull request SHA updater def release_job = job("generic-release-homebrew_pull_request_updater") OSRFUNIXBase.create(release_job) -GenericRemoteToken.create(release_job) include_common_params(release_job) release_job.with @@ -127,8 +126,6 @@ OSRFBrewCompilationAnyGitHub.create(bottle_job_builder, DISABLE_TESTS, NO_SUPPORTED_BRANCHES, DISABLE_GITHUB_INTEGRATION) -GenericRemoteToken.create(bottle_job_builder) - bottle_job_builder.with { wrappers { @@ -243,7 +240,6 @@ bottle_job_builder.with // 4. BREW bottle hash update def bottle_job_hash_updater = job(bottle_hash_updater_job_name) OSRFUNIXBase.create(bottle_job_hash_updater) -GenericRemoteToken.create(bottle_job_hash_updater) include_common_params(bottle_job_hash_updater) bottle_job_hash_updater.with diff --git a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl index 0d795f954..34342e25e 100644 --- a/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl +++ b/jenkins-scripts/dsl/gazebo_ros_pkgs.dsl @@ -260,7 +260,6 @@ bloom_debbuild_jobs.each { bloom_pkg -> // Use the linux install as base OSRFLinuxBuildPkgBase.create(build_pkg_job) - GenericRemoteToken.create(build_pkg_job) build_pkg_job.with { diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index b25e9d791..b8f110882 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -28,7 +28,6 @@ bridge_packages.each { pkg -> // Use the linux install as base OSRFLinuxBuildPkgBase.create(build_pkg_job) - GenericRemoteToken.create(build_pkg_job) build_pkg_job.with { From 9bef47294ccf66928ccc03a18c3a40ff1e5e44d9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 12 Nov 2024 20:29:18 +0100 Subject: [PATCH 02/12] Change job token auth by Jenkins user API token Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 10 +++-- release.py | 98 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 94 insertions(+), 14 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index 9d5c100c4..161319ce5 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -1,6 +1,8 @@ #!/bin/bash -e export _RELEASEPY_DEBUG=1 +export _RELEASEPY_TEST_CREDENTIALS=1 + test_dir=$(mktemp -d) export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release" mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian @@ -25,7 +27,7 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - gz-foo 1.2.3 token ${test_params} + gz-foo 1.2.3 ${test_params} } exec_ignition_releasepy_test() @@ -35,7 +37,7 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-foo 1.2.3 token ${test_params} + ign-foo 1.2.3 ${test_params} } exec_ignition_gazebo_releasepy_test() @@ -45,7 +47,7 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-gazebo 1.2.3 token ${test_params} + ign-gazebo 1.2.3 ${test_params} } exec_releasepy_with_real_gz() @@ -56,7 +58,7 @@ exec_releasepy_with_real_gz() --no-sanity-checks \ --source-repo-uri http://github.com/gazebosim/gz-common \ --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ - "${gz_pkg}" "${major_version}.x.y" token + "${gz_pkg}" "${major_version}.x.y" } expect_job_run() diff --git a/release.py b/release.py index 88708c84b..3f024c0ee 100755 --- a/release.py +++ b/release.py @@ -2,22 +2,25 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from configparser import ConfigParser from typing import Tuple +from urllib3.exceptions import RequestError +from urllib3.util import make_headers import subprocess import sys import tempfile import os import urllib.parse -import urllib.request +import urllib3 import argparse import shutil import venv -USAGE = 'release.py ' +USAGE = 'release.py ' try: JENKINS_URL = os.environ['JENKINS_URL'] except KeyError: - JENKINS_URL = 'http://build.osrfoundation.org' + JENKINS_URL = 'https://build.osrfoundation.org' JOB_NAME_PATTERN = '%s-debbuilder' GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' @@ -111,19 +114,18 @@ def parse_args(argv): Script to handle the release process for the Gazebo devs. Examples: A) Generate source: local repository tag + call source job: - $ release.py + $ release.py (auto calculate source-repo-uri from local directory) B) Call builders: reuse existing tarball version + call build jobs: - $ release.py --source-tarball-uri + $ release.py --source-tarball-uri (no call to source job, directly build jobs with tarball URL) C) Nightly builds (linux) - $ release.py --source-repo-existing-ref --upload-to-repo nightly + $ release.py --source-repo-existing-ref --upload-to-repo nightly """) parser.add_argument('package', help='which package to release') parser.add_argument('version', help='which version to release') - parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds') parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='dry-run; i.e., do actually run any of the commands') parser.add_argument('-a', '--package-alias', dest='package_alias', @@ -176,6 +178,50 @@ def parse_args(argv): return args +# +# BEGIN: Credentials code copied from ros_buildfarm +# +def get_credentials(jenkins_url=None): + try: + if os.environ['_RELEASEPY_TEST_CREDENTIALS']: + return 'fake_user', 'fake_api_token' + except KeyError: + pass + + config = ConfigParser() + config_file = get_credential_path() + if not os.path.exists(config_file): + print("Could not find credential file '%s'" % config_file, + file=sys.stderr) + return None, None + + config.read(config_file) + section_name = None + if jenkins_url is not None and jenkins_url in config: + section_name = jenkins_url + if section_name is None and 'DEFAULT' in config: + section_name = 'DEFAULT' + + if section_name is None or 'username' not in config[section_name] or \ + 'password' not in config[section_name]: + print( + "Could not find credentials for '%s' in file '%s'" % + (jenkins_url, config_file), file=sys.stderr) + return None, None + return config[section_name]['username'], config[section_name]['password'] + + +def get_credential_path(): + return os.path.join( + os.path.expanduser('~'), get_relative_credential_path()) + + +def get_relative_credential_path(): + return os.path.join('.buildfarm', 'jenkins.ini') + +# +# END: Credentials code copied from ros_buildfarm +# def get_release_repository_info(package): github_url = "https://github.com/gazebo-release/" + package + "-release" @@ -336,6 +382,13 @@ def sanity_checks(args, repo_dir): sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) + try: + if os.environ['_RELEASEPY_TEST_CREDENTIALS']: + pass + except KeyError: + check_credentials() + print_success("Jenkins credentials are good") + shutil.rmtree(repo_dir) @@ -486,6 +539,20 @@ def generate_source_params(args): return params +def build_credentials_header(): + username, api_token = get_credentials(JENKINS_URL) + if not username: + exit(1) + + return make_headers(basic_auth=f'{username}:{api_token}') + +def check_credentials(): + http = urllib3.PoolManager() + response = http.request('GET', JENKINS_URL, headers=build_credentials_header()) + if response.status != 200: + print(f"Crendentials error: {response.status}: {response.reason}") + http.clear() + exit(1) def call_jenkins_build(job_name, params, output_string, search_description_help): @@ -502,9 +569,21 @@ def call_jenkins_build(job_name, params, output_string, job_name, params_query) print_only_dbg(f" -- {output_string}: {url}") - if not DRY_RUN: - urllib.request.urlopen(url) + if not DRY_RUN: + http = urllib3.PoolManager() + try : + response = http.request('POST', url , headers=build_credentials_header()) + # 201 code is "created", it is the expected return of POST + if response.status != 201: + print(f"Error {response.status}: {response.reason}") + exit(1) + except RequestError as e: + print(f"An error occurred in the http request: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + finally: + http.clear() def display_help_job_chain_for_source_calls(args): # Encode the different ways using in the job descriptions to filter builds @@ -707,7 +786,6 @@ def go(argv): sanity_checks(args, repo_dir) params = generate_source_params(args) - params['token'] = args.jenkins_token params['PACKAGE'] = args.package params['VERSION'] = args.version if not NIGHTLY else 'nightly' params['RELEASE_REPO_BRANCH'] = args.release_repo_branch From a5f8094780f26b66435e95ba18c117fd9ff0f906 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 13 Nov 2024 13:13:18 +0100 Subject: [PATCH 03/12] Add an --auth parameter to specify user:token in the command line Signed-off-by: Jose Luis Rivero --- release.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/release.py b/release.py index 3f024c0ee..885d8f515 100755 --- a/release.py +++ b/release.py @@ -128,6 +128,9 @@ def parse_args(argv): parser.add_argument('version', help='which version to release') parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='dry-run; i.e., do actually run any of the commands') + parser.add_argument('--auth', dest='auth_input_arg', + default=None, + help='Explicit jenkins user:token string overriding the jenkins.ini credentials file.') parser.add_argument('-a', '--package-alias', dest='package_alias', default=None, help='different name that we are releasing under') @@ -386,7 +389,7 @@ def sanity_checks(args, repo_dir): if os.environ['_RELEASEPY_TEST_CREDENTIALS']: pass except KeyError: - check_credentials() + check_credentials(args.auth_input_arg) print_success("Jenkins credentials are good") shutil.rmtree(repo_dir) @@ -539,23 +542,31 @@ def generate_source_params(args): return params -def build_credentials_header(): - username, api_token = get_credentials(JENKINS_URL) - if not username: - exit(1) +def build_credentials_header(auth_input_arg = None): + if auth_input_arg: + username, api_token = auth_input_arg.split(':') + else: + username, api_token = get_credentials(JENKINS_URL) + if not username: + exit(1) return make_headers(basic_auth=f'{username}:{api_token}') -def check_credentials(): +def check_credentials(auth_input_arg = None): http = urllib3.PoolManager() - response = http.request('GET', JENKINS_URL, headers=build_credentials_header()) + response = http.request('GET', + JENKINS_URL, + headers=build_credentials_header(auth_input_arg)) if response.status != 200: print(f"Crendentials error: {response.status}: {response.reason}") http.clear() exit(1) -def call_jenkins_build(job_name, params, output_string, - search_description_help): +def call_jenkins_build(job_name, + params, + output_string, + search_description_help, + auth_input_arg = None): # Only to help user feedback this block help_url = f'{JENKINS_URL}/job/{job_name}' if search_description_help: @@ -573,7 +584,9 @@ def call_jenkins_build(job_name, params, output_string, if not DRY_RUN: http = urllib3.PoolManager() try : - response = http.request('POST', url , headers=build_credentials_header()) + response = http.request('POST', + url , + headers=build_credentials_header(auth_input_arg)) # 201 code is "created", it is the expected return of POST if response.status != 201: print(f"Error {response.status}: {response.reason}") From 87034411ec57b23f9500f49660b4c2307fe51f05 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 13 Nov 2024 13:16:05 +0100 Subject: [PATCH 04/12] Remove _RELEASEPY_TEST_CREDENTIALS and use --auth in test Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 5 ++++- release.py | 14 ++------------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index 161319ce5..814924955 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -1,7 +1,6 @@ #!/bin/bash -e export _RELEASEPY_DEBUG=1 -export _RELEASEPY_TEST_CREDENTIALS=1 test_dir=$(mktemp -d) export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release" @@ -27,6 +26,7 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ + --auth user:fake \ gz-foo 1.2.3 ${test_params} } @@ -37,6 +37,7 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ + --auth user:fake \ ign-foo 1.2.3 ${test_params} } @@ -47,6 +48,7 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ + --auth user:fake \ ign-gazebo 1.2.3 ${test_params} } @@ -56,6 +58,7 @@ exec_releasepy_with_real_gz() ./release.py \ --dry-run \ --no-sanity-checks \ + --auth user:fake \ --source-repo-uri http://github.com/gazebosim/gz-common \ --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ "${gz_pkg}" "${major_version}.x.y" diff --git a/release.py b/release.py index 885d8f515..90fdfc327 100755 --- a/release.py +++ b/release.py @@ -185,12 +185,6 @@ def parse_args(argv): # BEGIN: Credentials code copied from ros_buildfarm # def get_credentials(jenkins_url=None): - try: - if os.environ['_RELEASEPY_TEST_CREDENTIALS']: - return 'fake_user', 'fake_api_token' - except KeyError: - pass - config = ConfigParser() config_file = get_credential_path() if not os.path.exists(config_file): @@ -385,12 +379,8 @@ def sanity_checks(args, repo_dir): sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) - try: - if os.environ['_RELEASEPY_TEST_CREDENTIALS']: - pass - except KeyError: - check_credentials(args.auth_input_arg) - print_success("Jenkins credentials are good") + check_credentials(args.auth_input_arg) + print_success("Jenkins credentials are good") shutil.rmtree(repo_dir) From b96b5b632e1c9c55f155b17de6be7a1c2c0a46df Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 13 Nov 2024 13:20:26 +0100 Subject: [PATCH 05/12] Add input validation for user_token Signed-off-by: Jose Luis Rivero --- release.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release.py b/release.py index 90fdfc327..a6839fe0d 100755 --- a/release.py +++ b/release.py @@ -534,6 +534,8 @@ def generate_source_params(args): def build_credentials_header(auth_input_arg = None): if auth_input_arg: + if len(auth_input_arg.split(':')) != 2: + error("Auth string is not in the form of 'user:token' ") username, api_token = auth_input_arg.split(':') else: username, api_token = get_credentials(JENKINS_URL) From 1e31c437f39c026260d526b7d4855675d4f2d612 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 14 Nov 2024 11:08:00 +0100 Subject: [PATCH 06/12] Change DSL nightly call to use GITHUB_TOKEN --- .../dsl/_configs_/GitHubCredentials.groovy | 19 +++++++++++++++++++ jenkins-scripts/dsl/ignition_collection.dsl | 13 +++++++++---- 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy new file mode 100644 index 000000000..cb5bc374e --- /dev/null +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy @@ -0,0 +1,19 @@ +package _configs_ + +import javaposse.jobdsl.dsl.Job + +class GitHubCredentials +{ + static void createOsrfbuildToken(Job job) + { + job.with + { + wrappers { + // credential name needs to be in sync with provision code at infra/osrf-chef repo + credentialsBinding { + string('GITHUB_TOKEN', 'osrfbuild-token') + } + } + } + } +} diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index f214e5a9b..76c01480f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -148,6 +148,7 @@ nightly_collection = gz_collections_yaml.collections def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) +GitHubCredentials.createOsrfbuildToken(nightly_scheduler_job) nightly_scheduler_job.with { @@ -190,8 +191,6 @@ nightly_scheduler_job.with steps { shell("""\ #!/bin/bash -xe - set +x # keep password secret - PASS=\$(cat \$HOME/build_pass) dry_run_str="" if \$DRY_RUN; then @@ -239,9 +238,15 @@ nightly_scheduler_job.with src_branch="main" fi + set +x # safeguard keep password secret echo "releasing \${n} (from branch \${src_branch})" - python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly "\${PASS}" --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" - echo " - done" + # TODO: migrate the crendential to use USERNAME and PASSWORD instead only the token + # to avoid the hardcode of osrfbuild user here. + python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly \ + --auth 'osrfbuild:\${GITHUB_TOKEN}' \ + --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" + set -e + echo " - done (log is available)" done """.stripIndent()) From d607d77a89df54f382987f0511a8b1e2d55594c4 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Nov 2024 18:55:30 +0100 Subject: [PATCH 07/12] Implement test for checking osrfbuild credentials Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy index cb5bc374e..0e9a6ae38 100644 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy +++ b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy @@ -9,9 +9,9 @@ class GitHubCredentials job.with { wrappers { - // credential name needs to be in sync with provision code at infra/osrf-chef repo + // Credential name needs to be in sync with provision code at infra/osrf-chef repo credentialsBinding { - string('GITHUB_TOKEN', 'osrfbuild-token') + usernamePassword('OSRFBUILD_USER', 'OSRFBUILD_TOKEN', 'github-osrfbuild-credentials') } } } From 97bb45a1ac9046f647b5335401e89df52e17a6ea Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Nov 2024 14:00:35 +0100 Subject: [PATCH 08/12] update user/pass variables --- jenkins-scripts/dsl/ignition_collection.dsl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 76c01480f..cef5809cc 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -240,10 +240,8 @@ nightly_scheduler_job.with set +x # safeguard keep password secret echo "releasing \${n} (from branch \${src_branch})" - # TODO: migrate the crendential to use USERNAME and PASSWORD instead only the token - # to avoid the hardcode of osrfbuild user here. python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly \ - --auth 'osrfbuild:\${GITHUB_TOKEN}' \ + --auth "{OSRFBUILD_USER}:\${OSRFBUILD_TOKEN}"' \ --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" set -e echo " - done (log is available)" From c45b3666bdf222981fc92e102164c69f5fd64e0a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 14 Nov 2024 19:34:18 +0100 Subject: [PATCH 09/12] The _releasepy work also needs to be updated Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index 2dac2bce2..21181a71a 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -9,6 +9,7 @@ class OSRFReleasepy { // Base class for the job OSRFUNIXBase.create(job) + GitHubCredentials.createOsrfbuildToken(job) job.with { @@ -58,8 +59,6 @@ class OSRFReleasepy shell("""\ #!/bin/bash -xe - set +x # keep password secret - PASS=\$(cat \$HOME/build_pass) dry_run_str="" if \$DRY_RUN; then @@ -72,10 +71,10 @@ class OSRFReleasepy fi echo "releasing \${n} (from branch \${src_branch})" - python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" "\${PASS}" \${extra_osrf_repo} \ + python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" \${extra_osrf_repo} \ --source-tarball-uri \${SOURCE_TARBALL_URI} \ --release-repo-branch \${RELEASE_REPO_BRANCH} \ - --upload-to-repo \${UPLOAD_TO_REPO} > log + --upload-to-repo \${UPLOAD_TO_REPO} echo " - done" """.stripIndent()) } From fdc4bb0c9b12b1d966efbe8a8f32d975bc61dbed Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Nov 2024 19:10:12 +0100 Subject: [PATCH 10/12] The GitHubCrendentials class is replaced by OSRFCrendetials Signed-off-by: Jose Luis Rivero --- .../dsl/_configs_/GitHubCredentials.groovy | 19 ------------------- .../dsl/_configs_/OSRFReleasepy.groovy | 2 +- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 3 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy diff --git a/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy b/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy deleted file mode 100644 index 0e9a6ae38..000000000 --- a/jenkins-scripts/dsl/_configs_/GitHubCredentials.groovy +++ /dev/null @@ -1,19 +0,0 @@ -package _configs_ - -import javaposse.jobdsl.dsl.Job - -class GitHubCredentials -{ - static void createOsrfbuildToken(Job job) - { - job.with - { - wrappers { - // Credential name needs to be in sync with provision code at infra/osrf-chef repo - credentialsBinding { - usernamePassword('OSRFBUILD_USER', 'OSRFBUILD_TOKEN', 'github-osrfbuild-credentials') - } - } - } - } -} diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index 21181a71a..6d72f1930 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -9,7 +9,7 @@ class OSRFReleasepy { // Base class for the job OSRFUNIXBase.create(job) - GitHubCredentials.createOsrfbuildToken(job) + OSRFCredentials.setOSRFCrendentials(job, ['OSRFBUILD_JENKINS_TOKEN']) job.with { diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index cef5809cc..cd30a436f 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -148,7 +148,7 @@ nightly_collection = gz_collections_yaml.collections def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) -GitHubCredentials.createOsrfbuildToken(nightly_scheduler_job) +OSRFCredentials.setOSRFCrendentials(job, ['OSRFBUILD_JENKINS_TOKEN']) nightly_scheduler_job.with { From b46c4185800ec5e2c99c89b35ec41bce60ecdda5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Nov 2024 19:34:56 +0100 Subject: [PATCH 11/12] Missing auth and permissions to osrfbuild Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy | 2 ++ jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy | 1 + jenkins-scripts/dsl/brew_release.dsl | 1 + jenkins-scripts/dsl/ignition_collection.dsl | 10 +++++----- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy index 97ad235a3..fa100e6d9 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFLinuxBuildPkg.groovy @@ -2,6 +2,7 @@ package _configs_ import javaposse.jobdsl.dsl.Job import _configs_.Globals +import _configs_.OSRFCredentials /* -> OSRFLinuxBuildPkgBase @@ -27,6 +28,7 @@ class OSRFLinuxBuildPkg static void create(Job job, Map default_params = [:]) { OSRFLinuxBuildPkgBase.create(job) + OSRFCredentials.allowOsrfbuildToRunTheBuild(job) job.with { diff --git a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy index 6d72f1930..203dade3c 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFReleasepy.groovy @@ -72,6 +72,7 @@ class OSRFReleasepy echo "releasing \${n} (from branch \${src_branch})" python3 ./scripts/release.py \${dry_run_str} "\${PACKAGE}" "\${VERSION}" \${extra_osrf_repo} \ + --auth "\${OSRFBUILD_USER}:\${OSRFBUILD_TOKEN}"' \ --source-tarball-uri \${SOURCE_TARBALL_URI} \ --release-repo-branch \${RELEASE_REPO_BRANCH} \ --upload-to-repo \${UPLOAD_TO_REPO} diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 3644b0c31..1e6b79681 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -51,6 +51,7 @@ void include_common_params(Job job) // 1. BREW pull request SHA updater def release_job = job("generic-release-homebrew_pull_request_updater") OSRFUNIXBase.create(release_job) +OSRFCredentials.allowOsrfbuildToRunTheBuild(job) include_common_params(release_job) release_job.with diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index cd30a436f..7cc1d575c 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -238,13 +238,13 @@ nightly_scheduler_job.with src_branch="main" fi - set +x # safeguard keep password secret echo "releasing \${n} (from branch \${src_branch})" python3 ./scripts/release.py \${dry_run_str} "\${n}" nightly \ - --auth "{OSRFBUILD_USER}:\${OSRFBUILD_TOKEN}"' \ - --release-repo-branch main --nightly-src-branch \${src_branch} --upload-to-repo nightly > log || echo "MARK_AS_UNSTABLE" - set -e - echo " - done (log is available)" + --auth "\${OSRFBUILD_USER}:\${OSRFBUILD_TOKEN}"' \ + --release-repo-branch main \ + --nightly-src-branch \${src_branch} \ + --upload-to-repo nightly + echo " - done" done """.stripIndent()) From 7e392cb035197a96a97ca2bdcc23bea255b2dc5e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 25 Nov 2024 14:23:54 +0100 Subject: [PATCH 12/12] Fix typo Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/brew_release.dsl | 2 +- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 1e6b79681..c467c6fe4 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -51,7 +51,7 @@ void include_common_params(Job job) // 1. BREW pull request SHA updater def release_job = job("generic-release-homebrew_pull_request_updater") OSRFUNIXBase.create(release_job) -OSRFCredentials.allowOsrfbuildToRunTheBuild(job) +OSRFCredentials.allowOsrfbuildToRunTheBuild(release_job) include_common_params(release_job) release_job.with diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 7cc1d575c..fabdbfb2a 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -148,7 +148,7 @@ nightly_collection = gz_collections_yaml.collections def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) -OSRFCredentials.setOSRFCrendentials(job, ['OSRFBUILD_JENKINS_TOKEN']) +OSRFCredentials.setOSRFCrendentials(nightly_scheduler_job, ['OSRFBUILD_JENKINS_TOKEN']) nightly_scheduler_job.with {