From ae3790e3e831494987e7b4bff5ab9f3a04484929 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 16 Jun 2022 15:45:51 -0700 Subject: [PATCH 01/17] Migrate homebrew simulation and DSL replacements in bump_dependency Signed-off-by: methylDragon --- source-repo-scripts/bump_dependency.bash | 30 ++++++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index d7e86274c..3210e7b05 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -347,8 +347,10 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do cd ${TEMP_DIR}/homebrew-simulation startFromCleanBranch ${BUMP_BRANCH} master + # NOTE(CH3): This won't interfere with $LIB=gzXXX so it's fine # expand ign-* to ignition-* FORMULA_BASE=${LIB/ign/ignition} + # construct path with major version suffix FORMULA="Formula/${FORMULA_BASE}${VER}.rb" if [ ! -f "$FORMULA" ]; then @@ -358,6 +360,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Collection if ! [[ $VER == ?(-)+([0-9]) ]] ; then + # TODO(CH3): Change this to gz once we migrate the homebrew formulae cp Formula/ignition-${PREV_COLLECTION}.rb $FORMULA else cp Formula/${FORMULA_BASE}${PREV_VER}.rb $FORMULA @@ -369,13 +372,21 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do echo -e "${GREEN}${LIB}: Updating ${FORMULA}${DEFAULT}" URL="https://github.com/${ORG}/${LIB}.git" + # TOOD(CH3): Deprecated. Remove on tock? + # Support ign and ignition # libN sed -i -E "s ((${LIB#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA sed -i -E "s ((${LIB_#"ign_"}))${PREV_VER} \1${VER} g" $FORMULA - # ign-libN -> main - sed -i "s ${LIB}${PREV_VER} main g" $FORMULA # class IgnitionLibN sed -i -E "s/((class Ignition.*))${PREV_VER}/\1${VER}/g" $FORMULA + + # libN + sed -i -E "s ((${LIB#"gz-"}))${PREV_VER} \1${VER} g" $FORMULA + sed -i -E "s ((${LIB_#"gz_"}))${PREV_VER} \1${VER} g" $FORMULA + # gz-libN -> main + sed -i "s ${LIB}${PREV_VER} main g" $FORMULA + # class GzLibN + sed -i -E "s/((class Gz.*))${PREV_VER}/\1${VER}/g" $FORMULA sed -i -E "s/((class Sdformat))${PREV_VER}/\1${VER}/g" $FORMULA # remove bottle - TODO: this is only needed for new formulae sed -i -e "/bottle do/,/end/d" $FORMULA @@ -394,7 +405,11 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} + + # TODO(CH3): Deprecated. Remove on tock? + DEP_LIB=${DEP_LIB[$j]#"ign-"} + DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -441,6 +456,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Check if main branch of that library is the correct version PROJECT_NAME="${LIB_}${VER}" + # NOTE(CH3): This won't interfere with $LIB=gzXXX so it's fine PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" PROJECT="project.*(${PROJECT_NAME}" if ! grep -q ${PROJECT} "CMakeLists.txt"; then @@ -451,7 +467,11 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do echo -e "${GREEN}${LIB}: Updating source code${DEFAULT}" for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} + + # TODO(CH3): Deprecated. Remove on tock? + DEP_LIB=${DEP_LIB[$j]#"ign-"} + DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -475,7 +495,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" # Replace collection yaml branch names with main - if [[ "${LIB}" == "ign-${COLLECTION}" ]]; then + if [[ "${LIB}" == "ign-${COLLECTION}" || "${LIB}" == "gz-${COLLECTION}" ]]; then find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" fi From 90c072a8cb0f6584e38e5086c0e1ae209e2bc104 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 23 Jun 2022 17:47:33 -0700 Subject: [PATCH 02/17] TRIAL: Migrate ign_collection_name, ign_sw DSL vars Signed-off-by: methylDragon --- jenkins-scripts/dsl/ignition.dsl | 158 ++++++++++---------- jenkins-scripts/dsl/ignition_collection.dsl | 38 ++--- jenkins-scripts/lib/dependencies_archive.sh | 2 +- source-repo-scripts/bump_dependency.bash | 62 +++----- 4 files changed, 120 insertions(+), 140 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 2c8fc63e5..577a70afd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -130,9 +130,9 @@ String ci_distro_str = ci_distro[0] // Map of lists to use in CIWorkflow ci_pr_any_list = [:] -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> def list_empty = [] - ci_pr_any_list[ign_sw] = list_empty + ci_pr_any_list[gz_sw] = list_empty } /** @@ -317,24 +317,24 @@ boolean is_a_colcon_package(String ign_software_name) // ABI Checker job // Need to be before the ci-pr_any so the abi job name is defined -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> abi_distro.each { distro -> supported_arches.each { arch -> // Packages without ABI - if (ign_sw == 'tools' || ign_sw == 'cmake') + if (gz_sw == 'tools' || gz_sw == 'cmake') return - abi_job_names[ign_sw] = "ignition_${ign_sw}-abichecker-any_to_any-ubuntu_auto-${arch}" - def abi_job = job(abi_job_names[ign_sw]) - checkout_subdir = "ign-${ign_sw}" + abi_job_names[gz_sw] = "ignition_${gz_sw}-abichecker-any_to_any-ubuntu_auto-${arch}" + def abi_job = job(abi_job_names[gz_sw]) + checkout_subdir = "ign-${gz_sw}" OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, - "gazebosim/ign-${ign_sw}", - all_branches(ign_sw) - [ 'main']) + "gazebosim/ign-${gz_sw}", + all_branches(gz_sw) - [ 'main']) abi_job.with { extra_str="" - if (ign_sw == 'physics') + if (gz_sw == 'physics') { label "huge-memory" // on ARM native nodes in buildfarm we need to restrict to 1 the @@ -365,20 +365,20 @@ ignition_software.each { ign_sw -> } // end of ignition // MAIN CI JOBS (check every 5 minutes) -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // 1. Create the any job - def ignition_ci_job_name = "ignition_${ign_sw}-ci-pr_any-ubuntu_auto-${arch}" + def ignition_ci_job_name = "ignition_${gz_sw}-ci-pr_any-ubuntu_auto-${arch}" def ignition_ci_any_job = job(ignition_ci_job_name) - def ignition_checkout_dir = "ign-${ign_sw}" + def ignition_checkout_dir = "ign-${gz_sw}" OSRFLinuxCompilationAnyGitHub.create(ignition_ci_any_job, "gazebosim/${ignition_checkout_dir}", - enable_testing(ign_sw)) - include_gpu_label_if_needed(ignition_ci_any_job, ign_sw) + enable_testing(gz_sw)) + include_gpu_label_if_needed(ignition_ci_any_job, gz_sw) ignition_ci_any_job.with { - if (ign_sw == 'physics') + if (gz_sw == 'physics') label "huge-memory" steps @@ -392,29 +392,29 @@ ignition_software.each { ign_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_ci_job_name + ci_pr_any_list[gz_sw] << ignition_ci_job_name } } // INSTALL PACKAGE ALL PLATFORMS / DAILY -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> // Exclusion list - if (ign_sw in ignition_no_pkg_yet) + if (gz_sw in ignition_no_pkg_yet) return supported_arches.each { arch -> - supported_install_pkg_branches(ign_sw).each { major_version, supported_distros -> + supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> supported_distros.each { distro -> extra_repos_str="" - if ((ign_sw in ignition_prerelease_pkgs) && - (major_version in ignition_prerelease_pkgs[ign_sw]) && - (distro in ignition_prerelease_pkgs[ign_sw][major_version])) + if ((gz_sw in ignition_prerelease_pkgs) && + (major_version in ignition_prerelease_pkgs[gz_sw]) && + (distro in ignition_prerelease_pkgs[gz_sw][major_version])) extra_repos_str="prerelease" // No 1-dev or 0-dev packages (except special cases see @@ -423,9 +423,9 @@ ignition_software.each { ign_sw -> major_version = "" // -------------------------------------------------------------- - def install_default_job = job("ignition_${ign_sw}${major_version}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, ign_sw) + include_gpu_label_if_needed(install_default_job, gz_sw) install_default_job.with { @@ -433,8 +433,8 @@ ignition_software.each { ign_sw -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "libignition-${ign_sw}${major_version}-dev" - def gzdev_project = "ignition-${ign_sw}${major_version}" + def dev_package = "libignition-${gz_sw}${major_version}-dev" + def gzdev_project = "ignition-${gz_sw}${major_version}" steps { shell("""\ @@ -455,25 +455,25 @@ ignition_software.each { ign_sw -> } } -void generate_asan_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) +void generate_asan_ci_job(ignition_ci_job, gz_sw, branch, distro, arch) { - generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, + generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, '-DIGN_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } -void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, +void generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, extra_cmake = '', extra_test = '') { - OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(ign_sw)) + OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(gz_sw)) OSRFGitHub.create(ignition_ci_job, - "gazebosim/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") + "gazebosim/ign-${gz_sw}", + "${branch}", "ign-${gz_sw}") - include_gpu_label_if_needed(ignition_ci_job, ign_sw) + include_gpu_label_if_needed(ignition_ci_job, gz_sw) ignition_ci_job.with { - if (ign_sw == 'physics') + if (gz_sw == 'physics') label "huge-memory" steps { @@ -485,22 +485,22 @@ void generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${ign_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw}-compilation.bash """.stripIndent()) } } } // OTHER CI SUPPORTED JOBS -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> all_supported_distros.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily - all_branches("${ign_sw}").each { branch -> + all_branches("${gz_sw}").each { branch -> // 1. Standard CI - def ignition_ci_job = job("ignition_${ign_sw}-ci-${branch}-${distro}-${arch}") - generate_ci_job(ignition_ci_job, ign_sw, branch, distro, arch) + def ignition_ci_job = job("ignition_${gz_sw}-ci-${branch}-${distro}-${arch}") + generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch) ignition_ci_job.with { triggers { @@ -508,8 +508,8 @@ ignition_software.each { ign_sw -> } } // 2. ASAN CI - def ignition_ci_asan_job = job("ignition_${ign_sw}-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(ignition_ci_asan_job, ign_sw, branch, distro, arch) + def ignition_ci_asan_job = job("ignition_${gz_sw}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(ignition_ci_asan_job, gz_sw, branch, distro, arch) ignition_ci_asan_job.with { triggers { @@ -562,24 +562,24 @@ all_debbuilders().each { debbuilder_name -> // BREW: CI jobs // 1. any job -ignition_software.each { ign_sw -> - String ignition_brew_ci_any_job_name = "ignition_${ign_sw}-ci-pr_any-homebrew-amd64" +ignition_software.each { gz_sw -> + String ignition_brew_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-homebrew-amd64" def ignition_brew_ci_any_job = job(ignition_brew_ci_any_job_name) OSRFBrewCompilationAnyGitHub.create(ignition_brew_ci_any_job, - "gazebosim/ign-${ign_sw}", - enable_testing(ign_sw), + "gazebosim/ign-${gz_sw}", + enable_testing(gz_sw), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(ign_sw)) + enable_cmake_warnings(gz_sw)) ignition_brew_ci_any_job.with { steps { shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" @@ -593,17 +593,17 @@ ignition_software.each { ign_sw -> } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_brew_ci_any_job_name + ci_pr_any_list[gz_sw] << ignition_brew_ci_any_job_name // 2. main, release branches - all_branches("${ign_sw}").each { branch -> - def ignition_brew_ci_job = job("ignition_${ign_sw}-ci-${branch}-homebrew-amd64") + all_branches("${gz_sw}").each { branch -> + def ignition_brew_ci_job = job("ignition_${gz_sw}-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(ignition_brew_ci_job, - enable_testing(ign_sw), - enable_cmake_warnings(ign_sw)) + enable_testing(gz_sw), + enable_cmake_warnings(gz_sw)) OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/ign-${ign_sw}", - "${branch}", "ign-${ign_sw}") + "gazebosim/ign-${gz_sw}", + "${branch}", "ign-${gz_sw}") ignition_brew_ci_job.with { triggers { @@ -614,7 +614,7 @@ ignition_software.each { ign_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" @@ -629,21 +629,21 @@ ignition_software.each { ign_sw -> } // 3. install jobs to test bottles - supported_install_pkg_branches(ign_sw).each { major_version, supported_distros -> - def install_default_job = job("ignition_${ign_sw}${major_version}-install_bottle-homebrew-amd64") + supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> + def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with { // disable some bottles - if (("${ign_sw}" == "gui" && "${major_version}" == "0")) + if (("${gz_sw}" == "gui" && "${major_version}" == "0")) disabled() triggers { cron('@daily') } - def bottle_name = "ignition-${ign_sw}${major_version}" + def bottle_name = "ignition-${gz_sw}${major_version}" steps { shell("""\ @@ -671,15 +671,15 @@ ignition_software.each { ign_sw -> // WINDOWS: CI job // 1. any -ignition_software.each { ign_sw -> +ignition_software.each { gz_sw -> - if (is_a_colcon_package(ign_sw)) { + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - ignition_win_ci_any_job_name = "ign_${ign_sw}-pr-win" + ignition_win_ci_any_job_name = "ign_${gz_sw}-pr-win" Globals.gazebodistro_branch = true } else { - ignition_win_ci_any_job_name = "ignition_${ign_sw}-ci-pr_any-windows7-amd64" + ignition_win_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } @@ -699,37 +699,37 @@ ignition_software.each { ign_sw -> enable_testing(ign_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(ign_sw)) + enable_cmake_warnings(gz_sw)) ignition_win_ci_any_job.with { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[ign_sw] << ignition_win_ci_any_job_name + ci_pr_any_list[gz_sw] << ignition_win_ci_any_job_name // 2. main, release branches - all_branches("${ign_sw}").each { branch -> - if (is_a_colcon_package(ign_sw)) { + all_branches("${gz_sw}").each { branch -> + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal if (branch == 'main') branch_name = "ci" else - branch_name = branch - ign_sw - ignition_win_ci_job_name = "ign_${ign_sw}-${branch_name}-win" + branch_name = branch - gz_sw + ignition_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" } else { - ignition_win_ci_job_name = "ignition_${ign_sw}-ci-${branch}-windows7-amd64" + ignition_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" } def ignition_win_ci_job = job(ignition_win_ci_job_name) OSRFWinCompilation.create(ignition_win_ci_job, - enable_testing(ign_sw), - enable_cmake_warnings(ign_sw)) + enable_testing(gz_sw), + enable_cmake_warnings(gz_sw)) OSRFGitHub.create(ignition_win_ci_job, "gazebosim/gz-${ign_sw}", "${branch}") @@ -750,7 +750,7 @@ ignition_software.each { ign_sw -> steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${ign_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -758,8 +758,8 @@ ignition_software.each { ign_sw -> } // Main CI workflow -ignition_software.each { ign_sw -> - def String ci_main_name = "ignition_${ign_sw}-ci-manual_any" +ignition_software.each { gz_sw -> + def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" def ign_ci_main = pipelineJob(ci_main_name) - OSRFCIWorkFlowMultiAnyGitHub.create(ign_ci_main, ci_pr_any_list[ign_sw]) + OSRFCIWorkFlowMultiAnyGitHub.create(ign_ci_main, ci_pr_any_list[gz_sw]) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 93fa2eabb..328027348 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -292,27 +292,27 @@ ignition_collection_jobs = def DISABLE_TESTS = false // Testing compilation from source -ignition_collections.each { ign_collection -> +ignition_collections.each { gz_collection -> // COLCON - Windows - ign_collection_name = ign_collection.get('name') - def ignition_win_ci_job = job("ign_${ign_collection_name}-ci-win") + gz_collection_name = gz_collection.get('name') + def ignition_win_ci_job = job("ign_${gz_collection_name}-ci-win") Globals.gazebodistro_branch = true OSRFWinCompilation.create(ignition_win_ci_job, false) ignition_win_ci_job.with { steps { batchFile("""\ - set IGNITION_COLLECTION=${ign_collection_name} + set IGNITION_COLLECTION=${gz_collection_name} call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" """.stripIndent()) } } Globals.gazebodistro_branch = false - ign_collection.get('distros').each { distro -> + gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - def install_default_job = job("ignition_${ign_collection_name}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_collection_name}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) install_default_job.with @@ -321,7 +321,7 @@ ignition_collections.each { ign_collection -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "ignition-${ign_collection_name}" + def dev_package = "ignition-${gz_collection_name}" label "gpu-reliable" @@ -343,26 +343,26 @@ ignition_collections.each { ign_collection -> // MAC Brew CI job // -------------------------------------------------------------- - def ignition_brew_ci_job = job("ignition_${ign_collection_name}-ci-main-homebrew-amd64") - OSRFBrewCompilation.create(ignition_brew_ci_job, DISABLE_TESTS) - OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/ign-${ign_collection_name}", + def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/ign-${gz_collection_name/sim/gazebo}", "main", - "ign-${ign_collection_name}") - ignition_brew_ci_job.with + "ign-${gz_collection_name/sim/gazebo}") + gz_brew_ci_job.with { steps { shell("""\ #!/bin/bash -xe - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${ign_collection_name}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${gz_collection_name/sim/gazebo}" """.stripIndent()) } } // MAC Brew bottle install job // -------------------------------------------------------------- - def ignition_brew_install_bottle_job = job("ignition_${ign_collection_name}-install_bottle-homebrew-amd64") + def ignition_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(ignition_brew_install_bottle_job) ignition_brew_install_bottle_job.with @@ -371,7 +371,7 @@ ignition_collections.each { ign_collection -> cron('@daily') } - def bottle_name = "ignition-${ign_collection_name}" + def bottle_name = "ignition-${gz_collection_name}" steps { shell("""\ @@ -395,7 +395,7 @@ ignition_collections.each { ign_collection -> // DEBBUILD: linux package builder // -------------------------------------------------------------- - def build_pkg_job = job("ign-${ign_collection_name}-debbuilder") + def build_pkg_job = job("ign-${gz_collection_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with { @@ -412,10 +412,10 @@ ignition_collections.each { ign_collection -> dashboardView("ign-${ign_collection_name}") { jobs { - ignition_collection_jobs["${ign_collection_name}"].each { jobname -> + ignition_collection_jobs["${gz_collection_name}"].each { jobname -> name(jobname) } - if (ign_collection_name == ignition_nightly) { + if (gz_collection_name == ignition_nightly) { // add nightly debbuild jobs too ignition_collections.find { it.get('name') == ignition_nightly }.get('nightly_jobs').each { job -> name(job.getValue().get('debbuild') + '-debbuilder') diff --git a/jenkins-scripts/lib/dependencies_archive.sh b/jenkins-scripts/lib/dependencies_archive.sh index 6d0531ec9..3f2102cd6 100644 --- a/jenkins-scripts/lib/dependencies_archive.sh +++ b/jenkins-scripts/lib/dependencies_archive.sh @@ -549,7 +549,7 @@ if [[ ${IGN_GUI_MAJOR_VERSION} -lt 7 ]]; then fi fi -IGN_PHYSICS_DART_FROM_PKGS="true" +GZ_PHYSICS_DART_FROM_PKGS="true" # Completely rely on packages.apt from Garden if [[ ${IGN_PLUGIN_MAJOR_VERSION} -lt 2 ]]; then diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index 3210e7b05..e8d55d9b2 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -21,10 +21,10 @@ # Usage: # $ ./bump_dependency.bash ; ; [] # -# For example, to bump to `ign-rendering6` and all its dependencies, as well as +# For example, to bump to `gz-rendering6` and all its dependencies, as well as # `sdf12` and its dependencies on fortress using the `chapulina/fortress` branch for `docs`: # -# ./bump_dependency.bash fortress "ign-rendering;sdformat" "6;12" 428 edifice "chapulina/fortress" +# ./bump_dependency.bash fortress "gz-rendering;sdformat" "6;12" 428 edifice "chapulina/fortress" # # The `docs_branch` parameter is optional and defaults to `master` if not specified. # @@ -47,7 +47,7 @@ WHITE_BG="\e[107m" BLUE_BG="\e[44m" GREEN_BG="\e[42m" -IGN_ORG="gazebosim" +GZ_ORG="gazebosim" OSRF_ORG="osrf" TOOLING_ORG="gazebo-tooling" RELEASE_ORG="gazebo-release" @@ -211,7 +211,7 @@ cloneIfNeeded ${TOOLING_ORG} gazebodistro startFromCleanBranch bump_${COLLECTION} master # docs -cloneIfNeeded ${IGN_ORG} docs +cloneIfNeeded ${GZ_ORG} docs startFromCleanBranch bump_${COLLECTION} ${DOCS_BRANCH} # homebrew @@ -296,13 +296,13 @@ cd ${TEMP_DIR}/gazebodistro git reset --hard # Add collection to libraries, without version -LIBRARIES+=(ign-$COLLECTION) +LIBRARIES+=(gz-$COLLECTION) ################## # docs ################## cd ${TEMP_DIR}/docs -commitAndPR ${IGN_ORG} ${DOCS_BRANCH} +commitAndPR ${GZ_ORG} ${DOCS_BRANCH} for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do @@ -311,8 +311,8 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do LIB_=${LIB//-/_} # For fuel_tools VER=${VERSIONS[$i]} PREV_VER="$((${VER}-1))" - LIB_UPPER=`echo ${LIB#"ign-"} | tr a-z A-Z` - ORG=${IGN_ORG} + LIB_UPPER=`echo ${LIB#"gz-"} | tr a-z A-Z` + ORG=${GZ_ORG} BUMP_BRANCH="ci_matching_branch/bump_${COLLECTION}_${LIB}${VER}" echo -e "${BLUE_BG}Processing [${LIB}]${DEFAULT_BG}" @@ -329,7 +329,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do for ((j = 0; j < "${#LIBRARIES[@]}"; j++)); do - DEP_LIB=${LIBRARIES[$j]#"ign-"} + DEP_LIB=${LIBRARIES[$j]#"gz-"} DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -347,10 +347,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do cd ${TEMP_DIR}/homebrew-simulation startFromCleanBranch ${BUMP_BRANCH} master - # NOTE(CH3): This won't interfere with $LIB=gzXXX so it's fine - # expand ign-* to ignition-* - FORMULA_BASE=${LIB/ign/ignition} - # construct path with major version suffix FORMULA="Formula/${FORMULA_BASE}${VER}.rb" if [ ! -f "$FORMULA" ]; then @@ -372,14 +368,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do echo -e "${GREEN}${LIB}: Updating ${FORMULA}${DEFAULT}" URL="https://github.com/${ORG}/${LIB}.git" - # TOOD(CH3): Deprecated. Remove on tock? - # Support ign and ignition - # libN - sed -i -E "s ((${LIB#"ign-"}))${PREV_VER} \1${VER} g" $FORMULA - sed -i -E "s ((${LIB_#"ign_"}))${PREV_VER} \1${VER} g" $FORMULA - # class IgnitionLibN - sed -i -E "s/((class Ignition.*))${PREV_VER}/\1${VER}/g" $FORMULA - # libN sed -i -E "s ((${LIB#"gz-"}))${PREV_VER} \1${VER} g" $FORMULA sed -i -E "s ((${LIB_#"gz_"}))${PREV_VER} \1${VER} g" $FORMULA @@ -407,9 +395,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do DEP_LIB=${LIBRARIES[$j]#"gz-"} - # TODO(CH3): Deprecated. Remove on tock? - DEP_LIB=${DEP_LIB[$j]#"ign-"} - DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" @@ -456,8 +441,6 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do # Check if main branch of that library is the correct version PROJECT_NAME="${LIB_}${VER}" - # NOTE(CH3): This won't interfere with $LIB=gzXXX so it's fine - PROJECT_NAME="${PROJECT_NAME/ign_/ignition-}" PROJECT="project.*(${PROJECT_NAME}" if ! grep -q ${PROJECT} "CMakeLists.txt"; then echo -e "${RED}Wrong project name on [CMakeLists.txt], looking for [$PROJECT_NAME].${DEFAULT}" @@ -469,34 +452,31 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do DEP_LIB=${LIBRARIES[$j]#"gz-"} - # TODO(CH3): Deprecated. Remove on tock? - DEP_LIB=${DEP_LIB[$j]#"ign-"} - DEP_VER=${VERSIONS[$j]} DEP_PREV_VER="$((${DEP_VER}-1))" - # Replace lines like "find_package(ignition-cmake2 2.0.0)" - # with "find_package(ignition-cmake3)" + # Replace lines like "find_package(gz-cmake2 2.0.0)" + # with "find_package(gz-cmake3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER} \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}@g" - # Replace lines like "ign_find_package(ignition-math6 VERSION 6.5.0)" - # with "ign_find_package(ignition-math7)" + # Replace lines like "gz_find_package(gz-math6 VERSION 6.5.0)" + # with "gz_find_package(gz-math7)" # Preserves other args and handles edge cases: - # like "ign_find_package(ignition-math6 VERSION 6.5.0 REQUIRED)" - # with "ign_find_package(ignition-math6 REQUIRED)" - # like "ign_find_package(ignition-math6 REQUIRED COMPONENTS VERSION 6.10 eigen3)" - # with "ign_find_package(ignition-math7 REQUIRED COMPONENTS eigen3)" + # like "gz_find_package(gz-math6 VERSION 6.5.0 REQUIRED)" + # with "gz_find_package(gz-math6 REQUIRED)" + # like "gz_find_package(gz-math6 REQUIRED COMPONENTS VERSION 6.10 eigen3)" + # with "gz_find_package(gz-math7 REQUIRED COMPONENTS eigen3)" find . -type f -name 'CMakeLists.txt' -print0 | xargs -0 sed -i "s@\(find_package.*${DEP_LIB}\)${DEP_PREV_VER}\(.*\) \+VERSION \+${DEP_PREV_VER}[^ )]*@\1${DEP_VER}\2@g" # Rule: *plugin2 -> *plugin3 - # Replace lines like: "find_package(ignition-cmake2)" - # with: "find_package(ignition-cmake3)" + # Replace lines like: "find_package(gz-cmake2)" + # with: "find_package(gz-cmake3)" find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" # Replace collection yaml branch names with main - if [[ "${LIB}" == "ign-${COLLECTION}" || "${LIB}" == "gz-${COLLECTION}" ]]; then - find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s ign-${DEP_LIB}${DEP_VER} main g" + if [[ s"${LIB}" == "gz-${COLLECTION}" ]]; then + find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s gz-${DEP_LIB}${DEP_VER} main g" fi # Second run with _ instead of -, to support multiple variations of fuel-tools From 1e8b18bcf30261f6b74ceac5085341989205c136 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Thu, 4 Aug 2022 13:09:07 -0700 Subject: [PATCH 03/17] Prevent cases of ign-sim Find: (ign(ition)?[-_])\$\{gz([^/|-|\}]*)\} Replace: $1${gz$3/sim/gazebo} Signed-off-by: methylDragon --- jenkins-scripts/dsl/ignition.dsl | 60 ++++++++++----------- jenkins-scripts/dsl/ignition_collection.dsl | 14 ++--- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 577a70afd..35c9fdffd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -324,12 +324,12 @@ ignition_software.each { gz_sw -> if (gz_sw == 'tools' || gz_sw == 'cmake') return - abi_job_names[gz_sw] = "ignition_${gz_sw}-abichecker-any_to_any-ubuntu_auto-${arch}" + abi_job_names[gz_sw] = "ignition_${gz_sw/sim/gazebo}-abichecker-any_to_any-ubuntu_auto-${arch}" def abi_job = job(abi_job_names[gz_sw]) - checkout_subdir = "ign-${gz_sw}" + checkout_subdir = "ign-${gz_sw/sim/gazebo}" OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, - "gazebosim/ign-${gz_sw}", + "gazebosim/ign-${gz_sw/sim/gazebo}", all_branches(gz_sw) - [ 'main']) abi_job.with { @@ -369,9 +369,9 @@ ignition_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // 1. Create the any job - def ignition_ci_job_name = "ignition_${gz_sw}-ci-pr_any-ubuntu_auto-${arch}" + def ignition_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-ubuntu_auto-${arch}" def ignition_ci_any_job = job(ignition_ci_job_name) - def ignition_checkout_dir = "ign-${gz_sw}" + def ignition_checkout_dir = "ign-${gz_sw/sim/gazebo}" OSRFLinuxCompilationAnyGitHub.create(ignition_ci_any_job, "gazebosim/${ignition_checkout_dir}", enable_testing(gz_sw)) @@ -392,7 +392,7 @@ ignition_software.each { gz_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw/sim/gazebo}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job @@ -423,7 +423,7 @@ ignition_software.each { gz_sw -> major_version = "" // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_sw/sim/gazebo}${major_version}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) include_gpu_label_if_needed(install_default_job, gz_sw) @@ -433,8 +433,8 @@ ignition_software.each { gz_sw -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "libignition-${gz_sw}${major_version}-dev" - def gzdev_project = "ignition-${gz_sw}${major_version}" + def dev_package = "libignition-${gz_sw/sim/gazebo}${major_version}-dev" + def gzdev_project = "ignition-${gz_sw/sim/gazebo}${major_version}" steps { shell("""\ @@ -467,8 +467,8 @@ void generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, { OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(gz_sw)) OSRFGitHub.create(ignition_ci_job, - "gazebosim/ign-${gz_sw}", - "${branch}", "ign-${gz_sw}") + "gazebosim/ign-${gz_sw/sim/gazebo}", + "${branch}", "ign-${gz_sw/sim/gazebo}") include_gpu_label_if_needed(ignition_ci_job, gz_sw) ignition_ci_job.with @@ -485,7 +485,7 @@ void generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw/sim/gazebo}-compilation.bash """.stripIndent()) } } @@ -499,7 +499,7 @@ ignition_software.each { gz_sw -> // branches CI job scm@daily all_branches("${gz_sw}").each { branch -> // 1. Standard CI - def ignition_ci_job = job("ignition_${gz_sw}-ci-${branch}-${distro}-${arch}") + def ignition_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-${distro}-${arch}") generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch) ignition_ci_job.with { @@ -508,7 +508,7 @@ ignition_software.each { gz_sw -> } } // 2. ASAN CI - def ignition_ci_asan_job = job("ignition_${gz_sw}-ci_asan-${branch}-${distro}-${arch}") + def ignition_ci_asan_job = job("ignition_${gz_sw/sim/gazebo}-ci_asan-${branch}-${distro}-${arch}") generate_asan_ci_job(ignition_ci_asan_job, gz_sw, branch, distro, arch) ignition_ci_asan_job.with { @@ -563,12 +563,12 @@ all_debbuilders().each { debbuilder_name -> // 1. any job ignition_software.each { gz_sw -> - String ignition_brew_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-homebrew-amd64" + String ignition_brew_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-homebrew-amd64" def ignition_brew_ci_any_job = job(ignition_brew_ci_any_job_name) OSRFBrewCompilationAnyGitHub.create(ignition_brew_ci_any_job, - "gazebosim/ign-${gz_sw}", + "gazebosim/ign-${gz_sw/sim/gazebo}", enable_testing(gz_sw), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, @@ -579,7 +579,7 @@ ignition_software.each { gz_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" @@ -597,13 +597,13 @@ ignition_software.each { gz_sw -> // 2. main, release branches all_branches("${gz_sw}").each { branch -> - def ignition_brew_ci_job = job("ignition_${gz_sw}-ci-${branch}-homebrew-amd64") + def ignition_brew_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(ignition_brew_ci_job, enable_testing(gz_sw), enable_cmake_warnings(gz_sw)) OSRFGitHub.create(ignition_brew_ci_job, - "gazebosim/ign-${gz_sw}", - "${branch}", "ign-${gz_sw}") + "gazebosim/ign-${gz_sw/sim/gazebo}", + "${branch}", "ign-${gz_sw/sim/gazebo}") ignition_brew_ci_job.with { triggers { @@ -614,7 +614,7 @@ ignition_software.each { gz_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" @@ -630,7 +630,7 @@ ignition_software.each { gz_sw -> // 3. install jobs to test bottles supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> - def install_default_job = job("ignition_${gz_sw}${major_version}-install_bottle-homebrew-amd64") + def install_default_job = job("ignition_${gz_sw/sim/gazebo}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with @@ -643,7 +643,7 @@ ignition_software.each { gz_sw -> cron('@daily') } - def bottle_name = "ignition-${gz_sw}${major_version}" + def bottle_name = "ignition-${gz_sw/sim/gazebo}${major_version}" steps { shell("""\ @@ -676,10 +676,10 @@ ignition_software.each { gz_sw -> if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - ignition_win_ci_any_job_name = "ign_${gz_sw}-pr-win" + ignition_win_ci_any_job_name = "ign_${gz_sw/sim/gazebo}-pr-win" Globals.gazebodistro_branch = true } else { - ignition_win_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-windows7-amd64" + ignition_win_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } @@ -704,7 +704,7 @@ ignition_software.each { gz_sw -> { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -721,9 +721,9 @@ ignition_software.each { gz_sw -> branch_name = "ci" else branch_name = branch - gz_sw - ignition_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" + ignition_win_ci_job_name = "ign_${gz_sw/sim/gazebo}-${branch_name}-win" } else { - ignition_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" + ignition_win_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-${branch}-windows7-amd64" } def ignition_win_ci_job = job(ignition_win_ci_job_name) @@ -750,7 +750,7 @@ ignition_software.each { gz_sw -> steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -759,7 +759,7 @@ ignition_software.each { gz_sw -> // Main CI workflow ignition_software.each { gz_sw -> - def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" + def String ci_main_name = "ignition_${gz_sw/sim/gazebo}-ci-manual_any" def ign_ci_main = pipelineJob(ci_main_name) OSRFCIWorkFlowMultiAnyGitHub.create(ign_ci_main, ci_pr_any_list[gz_sw]) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 328027348..83ff84a3d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -295,7 +295,7 @@ def DISABLE_TESTS = false ignition_collections.each { gz_collection -> // COLCON - Windows gz_collection_name = gz_collection.get('name') - def ignition_win_ci_job = job("ign_${gz_collection_name}-ci-win") + def ignition_win_ci_job = job("ign_${gz_collection_name/sim/gazebo}-ci-win") Globals.gazebodistro_branch = true OSRFWinCompilation.create(ignition_win_ci_job, false) ignition_win_ci_job.with @@ -312,7 +312,7 @@ ignition_collections.each { gz_collection -> gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_collection_name}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_collection_name/sim/gazebo}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) install_default_job.with @@ -321,7 +321,7 @@ ignition_collections.each { gz_collection -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "ignition-${gz_collection_name}" + def dev_package = "ignition-${gz_collection_name/sim/gazebo}" label "gpu-reliable" @@ -343,7 +343,7 @@ ignition_collections.each { gz_collection -> // MAC Brew CI job // -------------------------------------------------------------- - def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") + def gz_brew_ci_job = job("ignition_${gz_collection_name/sim/gazebo}-ci-main-homebrew-amd64") OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) OSRFGitHub.create(gz_brew_ci_job, "gazebosim/ign-${gz_collection_name/sim/gazebo}", @@ -362,7 +362,7 @@ ignition_collections.each { gz_collection -> // MAC Brew bottle install job // -------------------------------------------------------------- - def ignition_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") + def ignition_brew_install_bottle_job = job("ignition_${gz_collection_name/sim/gazebo}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(ignition_brew_install_bottle_job) ignition_brew_install_bottle_job.with @@ -371,7 +371,7 @@ ignition_collections.each { gz_collection -> cron('@daily') } - def bottle_name = "ignition-${gz_collection_name}" + def bottle_name = "ignition-${gz_collection_name/sim/gazebo}" steps { shell("""\ @@ -395,7 +395,7 @@ ignition_collections.each { gz_collection -> // DEBBUILD: linux package builder // -------------------------------------------------------------- - def build_pkg_job = job("ign-${gz_collection_name}-debbuilder") + def build_pkg_job = job("ign-${gz_collection_name/sim/gazebo}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with { From e3a684bdc90c068b0b6f4e84c57457e1b451a541 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Wed, 17 Aug 2022 12:46:42 -0700 Subject: [PATCH 04/17] Migrate even more internal ign_ ignition_ variables Signed-off-by: methylDragon --- jenkins-scripts/dsl/ignition.dsl | 294 ++++++++++---------- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 2 files changed, 149 insertions(+), 147 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 35c9fdffd..5ab5cf07f 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -2,7 +2,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job // IGNITION PACKAGES -ignition_software = [ 'cmake', +gz_software = [ 'cmake', 'common', 'fuel-tools', 'gazebo', @@ -18,12 +18,12 @@ ignition_software = [ 'cmake', 'transport', 'utils' ] // DESC: need gpu/display for tests -ignition_gpu = [ 'gazebo', +gz_gpu = [ 'gazebo', 'gui', 'rendering', 'sensors' ] // DESC: software does not support cmake warnings enabled -ignition_no_cmake_warnings = [ 'cmake', +gz_no_cmake_warnings = [ 'cmake', 'common', 'fuel-tools', 'gazebo', @@ -38,12 +38,12 @@ ignition_no_cmake_warnings = [ 'cmake', 'transport', 'utils' ] // DESC: software does not have tests -ignition_no_test = [ ] +gz_no_test = [ ] // DESC: major series supported and released. The branches get CI, install pkg // testing and debbuild job. -// No branches in ignition_branches means no released branches (only CI on +// No branches in gz_branches means no released branches (only CI on // main, ABI check, install pkg) -ignition_branches = [ 'cmake' : [ '2' ], +gz_branches = [ 'cmake' : [ '2' ], 'common' : [ '1', '3', '4' ], 'fuel-tools' : [ '1', '4', '7' ], 'gazebo' : [ '3', '6' ], @@ -76,35 +76,35 @@ gz_branches = [ 'cmake' : [ '3' ], // DESC: prerelease branches are managed as any other supported branches for // special cases different to major branches: get compilation CI on the branch // physics/sensors don't need to be included since they use main for gz11 -ignition_prerelease_branches = [] +gz_prerelease_branches = [] // DESC: versioned names to generate debbuild jobs for special cases that -// don't appear in ignition_branches (like nightly builders or 0-debbuild +// don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) -ignition_extra_debbuild = [ 'gazebo7', +gz_extra_debbuild = [ 'gazebo7', 'utils1'] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 // DESC: exclude ignition from generate any install testing job -ignition_no_pkg_yet = [ ] +gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should -// not appear in ignition_no_pkg_yet nor in ignition_branches -ignition_prerelease_pkgs = [ 'placeholder' : [ +// not appear in gz_no_pkg_yet nor in gz_branches +gz_prerelease_pkgs = [ 'placeholder' : [ '1': [ 'bionic' ]], - ] + ] // packages using colcon for windows compilation while migrating all them to // this solution -ignition_colcon_win = [ 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] +gz_colcon_win = [ 'common', + 'fuel-tools', + 'gazebo', + 'gui', + 'launch', + 'math', + 'msgs', + 'physics', + 'plugin', + 'rendering', + 'sensors', + 'tools', + 'transport', + 'utils' ] // Main platform using for quick CI def ci_distro = Globals.get_ci_distro() @@ -130,7 +130,7 @@ String ci_distro_str = ci_distro[0] // Map of lists to use in CIWorkflow ci_pr_any_list = [:] -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> def list_empty = [] ci_pr_any_list[gz_sw] = list_empty } @@ -157,15 +157,6 @@ Map merge_maps(Map[] sources) { // return major versions supported or empty if just 0,1 series under // -dev package. -ArrayList supported_ign_branches(String ign_software) -{ - major_versions_registered = ignition_branches["${ign_software}"] - - if (major_versions_registered == null) - return [ '' ] - - return major_versions_registered -} ArrayList supported_gz_branches(String gz_software) { major_versions_registered = gz_branches["${gz_software}"] @@ -177,9 +168,9 @@ ArrayList supported_gz_branches(String gz_software) } // return prerelease branch names -ArrayList prerelease_branches(String ign_software) +ArrayList prerelease_branches(String gz_software) { - pre_branches = ignition_prerelease_branches["${ign_software}"] + pre_branches = gz_prerelease_branches["${gz_software}"] if (pre_branches == null) return [ '' ] @@ -188,21 +179,21 @@ ArrayList prerelease_branches(String ign_software) } // return all ci branch names -ArrayList all_branches(String ign_software) +ArrayList all_branches(String gz_software) { List branches = new ArrayList(); - supported_ign_branches("${ign_software}").each { major_version -> + supported_gz_branches("${gz_software}").each { major_version -> if ("${major_version}") { - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${gz_software}${major_version}") } } - supported_gz_branches("${ign_software}").each { major_version -> + supported_gz_branches("${gz_software}").each { major_version -> if ("${major_version}") { - branches.add("gz-${ign_software}${major_version}") + branches.add("gz-${gz_software}${major_version}") } } branches.add('main') - prerelease_branches("${ign_software}").each { branch -> + prerelease_branches("${gz_software}").each { branch -> if ("${branch}") { branches.add(branch) } @@ -215,29 +206,29 @@ ArrayList all_debbuilders() { List branches = new ArrayList(); // add all supported branches - ignition_software.each { ign_software -> - supported_ign_branches("${ign_software}").each { major_version -> + gz_software.each { gz_software -> + supported_gz_branches("${gz_software}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${gz_software}${major_version}") } } - supported_gz_branches("${ign_software}").each { major_version -> + supported_gz_branches("${gz_software}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${ign_software}${major_version}") + branches.add("ign-${gz_software}${major_version}") } } } // add all extra debbuilders - ignition_extra_debbuild.each { ign_name -> - branches.add("ign-${ign_name}") + gz_extra_debbuild.each { gz_name -> + branches.add("ign-${gz_name}") } return branches @@ -247,34 +238,34 @@ ArrayList all_debbuilders() // Map with the form of: major versions as keys. // Lists of distros supported as values // @TODO[scpeters] update this to support gz_branches after garden release -Map supported_install_pkg_branches(String ign_software) +Map supported_install_pkg_branches(String gz_software) { - major_versions_prerelease = ignition_prerelease_pkgs["${ign_software}"] + major_versions_prerelease = gz_prerelease_pkgs["${gz_software}"] - // construct a map of stable packages based on supported_ign_branches and + // construct a map of stable packages based on supported_gz_branches and // all_supported_distros map_of_stable_versions = [:] - map_of_stable_versions[ign_software] = [:] - supported_ign_branches(ign_software).each { major_version -> + map_of_stable_versions[gz_software] = [:] + supported_gz_branches(gz_software).each { major_version -> new_relation = [:] new_relation[major_version] = all_supported_distros - map_of_stable_versions[ign_software] << new_relation + map_of_stable_versions[gz_software] << new_relation } if (major_versions_prerelease == null) - return map_of_stable_versions[ign_software]; + return map_of_stable_versions[gz_software]; - return merge_maps(map_of_stable_versions[ign_software], + return merge_maps(map_of_stable_versions[gz_software], major_versions_prerelease) } -void include_gpu_label_if_needed(Job job, String ign_software_name) +void include_gpu_label_if_needed(Job job, String gz_software_name) { job.with { - ignition_gpu.each { ign_each -> - if (ign_software_name == ign_each) + gz_gpu.each { gz_each -> + if (gz_software_name == gz_each) { label "gpu-reliable" @@ -291,25 +282,25 @@ void include_gpu_label_if_needed(Job job, String ign_software_name) } } -boolean enable_cmake_warnings(String ign_software_name) +boolean enable_cmake_warnings(String gz_software_name) { - if (ign_software_name in ignition_no_cmake_warnings) + if (gz_software_name in gz_no_cmake_warnings) return false return true } -boolean enable_testing(String ign_software_name) +boolean enable_testing(String gz_software_name) { - if (ign_software_name in ignition_no_test) + if (gz_software_name in gz_no_test) return false return true } -boolean is_a_colcon_package(String ign_software_name) +boolean is_a_colcon_package(String gz_software_name) { - if (ign_software_name in ignition_colcon_win) + if (gz_software_name in gz_colcon_win) return true return false @@ -317,20 +308,25 @@ boolean is_a_colcon_package(String ign_software_name) // ABI Checker job // Need to be before the ci-pr_any so the abi job name is defined -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> abi_distro.each { distro -> supported_arches.each { arch -> // Packages without ABI if (gz_sw == 'tools' || gz_sw == 'cmake') return - abi_job_names[gz_sw] = "ignition_${gz_sw/sim/gazebo}-abichecker-any_to_any-ubuntu_auto-${arch}" - def abi_job = job(abi_job_names[gz_sw]) - checkout_subdir = "ign-${gz_sw/sim/gazebo}" + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + abi_job_names[software_name] = "ignition_${software_name}-abichecker-any_to_any-ubuntu_auto-${arch}" + def abi_job = job(abi_job_names[software_name]) + checkout_subdir = "ign-${software_name}" OSRFLinuxABIGitHub.create(abi_job) GenericAnyJobGitHub.create(abi_job, - "gazebosim/ign-${gz_sw/sim/gazebo}", - all_branches(gz_sw) - [ 'main']) + "gazebosim/ign-${software_name}", + all_branches(software_name) - [ 'main']) abi_job.with { extra_str="" @@ -365,18 +361,23 @@ ignition_software.each { gz_sw -> } // end of ignition // MAIN CI JOBS (check every 5 minutes) -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // 1. Create the any job - def ignition_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-ubuntu_auto-${arch}" - def ignition_ci_any_job = job(ignition_ci_job_name) - def ignition_checkout_dir = "ign-${gz_sw/sim/gazebo}" - OSRFLinuxCompilationAnyGitHub.create(ignition_ci_any_job, - "gazebosim/${ignition_checkout_dir}", - enable_testing(gz_sw)) - include_gpu_label_if_needed(ignition_ci_any_job, gz_sw) - ignition_ci_any_job.with + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + def gz_ci_job_name = "ignition_${software_name}-ci-pr_any-ubuntu_auto-${arch}" + def gz_ci_any_job = job(gz_ci_job_name) + def gz_checkout_dir = "ign-${software_name}" + OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, + "gazebosim/${gz_checkout_dir}", + enable_testing(software_name)) + include_gpu_label_if_needed(gz_ci_any_job, software_name) + gz_ci_any_job.with { if (gz_sw == 'physics') label "huge-memory" @@ -392,40 +393,45 @@ ignition_software.each { gz_sw -> export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw/sim/gazebo}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash """.stripIndent()) } // end of steps } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[gz_sw] << ignition_ci_job_name + ci_pr_any_list[software_name] << gz_ci_job_name } } // INSTALL PACKAGE ALL PLATFORMS / DAILY -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> // Exclusion list - if (gz_sw in ignition_no_pkg_yet) + if (gz_sw in gz_no_pkg_yet) return supported_arches.each { arch -> supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> supported_distros.each { distro -> extra_repos_str="" - if ((gz_sw in ignition_prerelease_pkgs) && - (major_version in ignition_prerelease_pkgs[gz_sw]) && - (distro in ignition_prerelease_pkgs[gz_sw][major_version])) + if ((gz_sw in gz_prerelease_pkgs) && + (major_version in gz_prerelease_pkgs[gz_sw]) && + (distro in gz_prerelease_pkgs[gz_sw][major_version])) extra_repos_str="prerelease" // No 1-dev or 0-dev packages (except special cases see - // ignition_debbuild variable), unversioned + // gz_debbuild variable), unversioned if ("${major_version}" == "0" || "${major_version}" == "1") major_version = "" + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_sw/sim/gazebo}${major_version}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${software_name}${major_version}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, gz_sw) + include_gpu_label_if_needed(install_default_job, software_name) install_default_job.with { @@ -433,8 +439,8 @@ ignition_software.each { gz_sw -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "libignition-${gz_sw/sim/gazebo}${major_version}-dev" - def gzdev_project = "ignition-${gz_sw/sim/gazebo}${major_version}" + def dev_package = "libignition-${software_name}${major_version}-dev" + def gzdev_project = "ignition-${software_name}${major_version}" steps { shell("""\ @@ -455,23 +461,23 @@ ignition_software.each { gz_sw -> } } -void generate_asan_ci_job(ignition_ci_job, gz_sw, branch, distro, arch) +void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) { - generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, + generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, '-DIGN_SANITIZER=Address', Globals.MAKETEST_SKIP_IGN) } -void generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, +void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, extra_cmake = '', extra_test = '') { - OSRFLinuxCompilation.create(ignition_ci_job, enable_testing(gz_sw)) - OSRFGitHub.create(ignition_ci_job, + OSRFLinuxCompilation.create(gz_ci_job, enable_testing(gz_sw)) + OSRFGitHub.create(gz_ci_job, "gazebosim/ign-${gz_sw/sim/gazebo}", "${branch}", "ign-${gz_sw/sim/gazebo}") - include_gpu_label_if_needed(ignition_ci_job, gz_sw) - ignition_ci_job.with + include_gpu_label_if_needed(gz_ci_job, gz_sw) + gz_ci_job.with { if (gz_sw == 'physics') label "huge-memory" @@ -492,25 +498,25 @@ void generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch, } // OTHER CI SUPPORTED JOBS -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> all_supported_distros.each { distro -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily all_branches("${gz_sw}").each { branch -> // 1. Standard CI - def ignition_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-${distro}-${arch}") - generate_ci_job(ignition_ci_job, gz_sw, branch, distro, arch) - ignition_ci_job.with + def gz_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-${distro}-${arch}") + generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch) + gz_ci_job.with { triggers { scm('@daily') } } // 2. ASAN CI - def ignition_ci_asan_job = job("ignition_${gz_sw/sim/gazebo}-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(ignition_ci_asan_job, gz_sw, branch, distro, arch) - ignition_ci_asan_job.with + def gz_ci_asan_job = job("ignition_${gz_sw/sim/gazebo}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(gz_ci_asan_job, gz_sw, branch, distro, arch) + gz_ci_asan_job.with { triggers { scm(Globals.CRON_ON_WEEKEND) @@ -562,18 +568,18 @@ all_debbuilders().each { debbuilder_name -> // BREW: CI jobs // 1. any job -ignition_software.each { gz_sw -> - String ignition_brew_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-homebrew-amd64" +gz_software.each { gz_sw -> + String gz_brew_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-homebrew-amd64" - def ignition_brew_ci_any_job = job(ignition_brew_ci_any_job_name) - OSRFBrewCompilationAnyGitHub.create(ignition_brew_ci_any_job, + def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) + OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, "gazebosim/ign-${gz_sw/sim/gazebo}", enable_testing(gz_sw), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, enable_cmake_warnings(gz_sw)) - ignition_brew_ci_any_job.with + gz_brew_ci_any_job.with { steps { shell("""\ @@ -584,27 +590,25 @@ ignition_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${gz_sw/sim/gazebo}" fi """.stripIndent()) } } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[gz_sw] << ignition_brew_ci_any_job_name + ci_pr_any_list[gz_sw] << gz_brew_ci_any_job_name // 2. main, release branches all_branches("${gz_sw}").each { branch -> - def ignition_brew_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-homebrew-amd64") - OSRFBrewCompilation.create(ignition_brew_ci_job, + def gz_brew_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, enable_testing(gz_sw), enable_cmake_warnings(gz_sw)) - OSRFGitHub.create(ignition_brew_ci_job, + OSRFGitHub.create(gz_brew_ci_job, "gazebosim/ign-${gz_sw/sim/gazebo}", "${branch}", "ign-${gz_sw/sim/gazebo}") - ignition_brew_ci_job.with + gz_brew_ci_job.with { triggers { scm('@daily') @@ -619,9 +623,7 @@ ignition_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${gz_sw/sim/gazebo}" fi """.stripIndent()) } @@ -671,36 +673,36 @@ ignition_software.each { gz_sw -> // WINDOWS: CI job // 1. any -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - ignition_win_ci_any_job_name = "ign_${gz_sw/sim/gazebo}-pr-win" + gz_win_ci_any_job_name = "ign_${gz_sw/sim/gazebo}-pr-win" Globals.gazebodistro_branch = true } else { - ignition_win_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-windows7-amd64" + gz_win_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } supported_branches = [] // ign-gazebo only support Windows from ign-gazebo5 - if (ign_sw == 'gazebo') + if (gz_sw == 'gazebo') supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] // ign-launch only support Windows from ign-launch5 - if (ign_sw == 'launch') + if (gz_sw == 'launch') supported_branches = [ 'ign-launch5', 'gz-launch6', 'main' ] - def ignition_win_ci_any_job = job(ignition_win_ci_any_job_name) - OSRFWinCompilationAnyGitHub.create(ignition_win_ci_any_job, - "gazebosim/gz-${ign_sw}", - enable_testing(ign_sw), + def gz_win_ci_any_job = job(gz_win_ci_any_job_name) + OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, + "gazebosim/gz-${gz_sw/gazebo/sim}", + enable_testing(gz_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, enable_cmake_warnings(gz_sw)) - ignition_win_ci_any_job.with + gz_win_ci_any_job.with { steps { batchFile("""\ @@ -710,7 +712,7 @@ ignition_software.each { gz_sw -> } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[gz_sw] << ignition_win_ci_any_job_name + ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name // 2. main, release branches all_branches("${gz_sw}").each { branch -> @@ -721,20 +723,20 @@ ignition_software.each { gz_sw -> branch_name = "ci" else branch_name = branch - gz_sw - ignition_win_ci_job_name = "ign_${gz_sw/sim/gazebo}-${branch_name}-win" + gz_win_ci_job_name = "ign_${gz_sw/sim/gazebo}-${branch_name}-win" } else { - ignition_win_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-${branch}-windows7-amd64" + gz_win_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-${branch}-windows7-amd64" } - def ignition_win_ci_job = job(ignition_win_ci_job_name) - OSRFWinCompilation.create(ignition_win_ci_job, + def gz_win_ci_job = job(gz_win_ci_job_name) + OSRFWinCompilation.create(gz_win_ci_job, enable_testing(gz_sw), enable_cmake_warnings(gz_sw)) - OSRFGitHub.create(ignition_win_ci_job, + OSRFGitHub.create(gz_win_ci_job, "gazebosim/gz-${ign_sw}", "${branch}") - ignition_win_ci_job.with + gz_win_ci_job.with { // ign-gazebo only works on Windows from ign-gazebo5 if (branch == 'ign-gazebo3') @@ -758,8 +760,8 @@ ignition_software.each { gz_sw -> } // Main CI workflow -ignition_software.each { gz_sw -> +gz_software.each { gz_sw -> def String ci_main_name = "ignition_${gz_sw/sim/gazebo}-ci-manual_any" - def ign_ci_main = pipelineJob(ci_main_name) - OSRFCIWorkFlowMultiAnyGitHub.create(ign_ci_main, ci_pr_any_list[gz_sw]) + def gz_ci_main = pipelineJob(ci_main_name) + OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 83ff84a3d..a2df6a408 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -409,7 +409,7 @@ ignition_collections.each { gz_collection -> } // Gazebo dashboards - dashboardView("ign-${ign_collection_name}") + dashboardView("ign-${gz_collection_name}") { jobs { ignition_collection_jobs["${gz_collection_name}"].each { jobname -> From 877649194f809c63b29468515bbacf7a53323da0 Mon Sep 17 00:00:00 2001 From: methylDragon Date: Wed, 17 Aug 2022 15:17:13 -0700 Subject: [PATCH 05/17] Fix gz_sw Signed-off-by: methylDragon --- jenkins-scripts/dsl/ignition.dsl | 102 +++++++++++--------- jenkins-scripts/dsl/ignition_collection.dsl | 49 +++++----- 2 files changed, 83 insertions(+), 68 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 5ab5cf07f..6390a270d 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -410,7 +410,7 @@ gz_software.each { gz_sw -> return supported_arches.each { arch -> - supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> + supported_install_pkg_branches(software_name).each { major_version, supported_distros -> supported_distros.each { distro -> extra_repos_str="" if ((gz_sw in gz_prerelease_pkgs) && @@ -471,10 +471,10 @@ void generate_asan_ci_job(gz_ci_job, gz_sw, branch, distro, arch) void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, extra_cmake = '', extra_test = '') { - OSRFLinuxCompilation.create(gz_ci_job, enable_testing(gz_sw)) + OSRFLinuxCompilation.create(gz_ci_job, enable_testing(software_name)) OSRFGitHub.create(gz_ci_job, - "gazebosim/ign-${gz_sw/sim/gazebo}", - "${branch}", "ign-${gz_sw/sim/gazebo}") + "gazebosim/ign-${software_name}", + "${branch}", "ign-${software_name}") include_gpu_label_if_needed(gz_ci_job, gz_sw) gz_ci_job.with @@ -491,7 +491,7 @@ void generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch, export BUILDING_EXTRA_MAKETEST_PARAMS="${extra_test}" export DISTRO=${distro} export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${gz_sw/sim/gazebo}-compilation.bash + /bin/bash -xe ./scripts/jenkins-scripts/docker/ign_${software_name}-compilation.bash """.stripIndent()) } } @@ -503,10 +503,15 @@ gz_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily - all_branches("${gz_sw}").each { branch -> + all_branches("${software_name}").each { branch -> // 1. Standard CI - def gz_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-${distro}-${arch}") - generate_ci_job(gz_ci_job, gz_sw, branch, distro, arch) + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + def gz_ci_job = job("ignition_${software_name}-ci-${branch}-${distro}-${arch}") + generate_ci_job(gz_ci_job, software_name, branch, distro, arch) gz_ci_job.with { triggers { @@ -514,8 +519,8 @@ gz_software.each { gz_sw -> } } // 2. ASAN CI - def gz_ci_asan_job = job("ignition_${gz_sw/sim/gazebo}-ci_asan-${branch}-${distro}-${arch}") - generate_asan_ci_job(gz_ci_asan_job, gz_sw, branch, distro, arch) + def gz_ci_asan_job = job("ignition_${software_name}-ci_asan-${branch}-${distro}-${arch}") + generate_asan_ci_job(gz_ci_asan_job, software_name, branch, distro, arch) gz_ci_asan_job.with { triggers { @@ -569,45 +574,50 @@ all_debbuilders().each { debbuilder_name -> // 1. any job gz_software.each { gz_sw -> - String gz_brew_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-homebrew-amd64" + software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + + if (gz_sw == 'sim') + software_name = "gazebo" + + String gz_brew_ci_any_job_name = "ignition_${software_name}-ci-pr_any-homebrew-amd64" def gz_brew_ci_any_job = job(gz_brew_ci_any_job_name) OSRFBrewCompilationAnyGitHub.create(gz_brew_ci_any_job, - "gazebosim/ign-${gz_sw/sim/gazebo}", - enable_testing(gz_sw), + "gazebosim/ign-${software_name}", + enable_testing(software_name), GITHUB_SUPPORT_ALL_BRANCHES, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(gz_sw)) + enable_cmake_warnings(software_name)) gz_brew_ci_any_job.with { steps { shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${gz_sw/sim/gazebo}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${software_name}" fi """.stripIndent()) } } // add ci-pr_any to the list for CIWorkflow - ci_pr_any_list[gz_sw] << gz_brew_ci_any_job_name + ci_pr_any_list[software_name] << gz_brew_ci_any_job_name // 2. main, release branches - all_branches("${gz_sw}").each { branch -> - def gz_brew_ci_job = job("ignition_${gz_sw/sim/gazebo}-ci-${branch}-homebrew-amd64") + all_branches("${software_name}").each { branch -> + def gz_brew_ci_job = job("ignition_${software_name}-ci-${branch}-homebrew-amd64") OSRFBrewCompilation.create(gz_brew_ci_job, - enable_testing(gz_sw), - enable_cmake_warnings(gz_sw)) + enable_testing(software_name), + enable_cmake_warnings(software_name)) OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/ign-${gz_sw/sim/gazebo}", - "${branch}", "ign-${gz_sw/sim/gazebo}") + "gazebosim/ign-${software_name}", + "${branch}", "ign-${software_name}") gz_brew_ci_job.with { triggers { @@ -618,12 +628,12 @@ gz_software.each { gz_sw -> shell("""\ #!/bin/bash -xe - export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-homebrew-amd64.bash" + export HOMEBREW_SCRIPT="./scripts/jenkins-scripts/ign_${software_name}-default-devel-homebrew-amd64.bash" if [ -s "\$HOMEBREW_SCRIPT" ] then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${gz_sw/sim/gazebo}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${software_name}" fi """.stripIndent()) } @@ -631,8 +641,8 @@ gz_software.each { gz_sw -> } // 3. install jobs to test bottles - supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> - def install_default_job = job("ignition_${gz_sw/sim/gazebo}${major_version}-install_bottle-homebrew-amd64") + supported_install_pkg_branches(software_name).each { major_version, supported_distros -> + def install_default_job = job("ignition_${software_name}${major_version}-install_bottle-homebrew-amd64") OSRFBrewInstall.create(install_default_job) install_default_job.with @@ -645,7 +655,7 @@ gz_software.each { gz_sw -> cron('@daily') } - def bottle_name = "ignition-${gz_sw/sim/gazebo}${major_version}" + def bottle_name = "ignition-${software_name}${major_version}" steps { shell("""\ @@ -675,21 +685,25 @@ gz_software.each { gz_sw -> // 1. any gz_software.each { gz_sw -> - if (is_a_colcon_package(gz_sw)) { + if (is_a_colcon_package(software_name)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - gz_win_ci_any_job_name = "ign_${gz_sw/sim/gazebo}-pr-win" + gz_win_ci_any_job_name = "ign_${software_name}-pr-win" Globals.gazebodistro_branch = true } else { - gz_win_ci_any_job_name = "ignition_${gz_sw/sim/gazebo}-ci-pr_any-windows7-amd64" + gz_win_ci_any_job_name = "ignition_${software_name}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } supported_branches = [] + gz_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite + ign_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite // ign-gazebo only support Windows from ign-gazebo5 if (gz_sw == 'gazebo') supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] + ign_software_name = "gazebo" + gz_software_name = "sim" // ign-launch only support Windows from ign-launch5 if (gz_sw == 'launch') @@ -697,16 +711,16 @@ gz_software.each { gz_sw -> def gz_win_ci_any_job = job(gz_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, - "gazebosim/gz-${gz_sw/gazebo/sim}", - enable_testing(gz_sw), + "gazebosim/gz-${software_name}", + enable_testing(software_name), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(gz_sw)) + enable_cmake_warnings(software_name)) gz_win_ci_any_job.with { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${ign_software_name}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -715,25 +729,25 @@ gz_software.each { gz_sw -> ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name // 2. main, release branches - all_branches("${gz_sw}").each { branch -> - if (is_a_colcon_package(gz_sw)) { + all_branches("${software_name}").each { branch -> + if (is_a_colcon_package(software_name)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal if (branch == 'main') branch_name = "ci" else branch_name = branch - gz_sw - gz_win_ci_job_name = "ign_${gz_sw/sim/gazebo}-${branch_name}-win" + gz_win_ci_job_name = "ign_${ign_software_name}-${branch_name}-win" } else { - gz_win_ci_job_name = "ignition_${gz_sw/sim/gazebo}-ci-${branch}-windows7-amd64" + gz_win_ci_job_name = "ignition_${ign_software_name}-ci-${branch}-windows7-amd64" } def gz_win_ci_job = job(gz_win_ci_job_name) OSRFWinCompilation.create(gz_win_ci_job, - enable_testing(gz_sw), - enable_cmake_warnings(gz_sw)) + enable_testing(software_name), + enable_cmake_warnings(software_name)) OSRFGitHub.create(gz_win_ci_job, - "gazebosim/gz-${ign_sw}", + "gazebosim/gz-${software_name}", "${branch}") gz_win_ci_job.with @@ -752,7 +766,7 @@ gz_software.each { gz_sw -> steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_sw/sim/gazebo}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${software_name}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -761,7 +775,7 @@ gz_software.each { gz_sw -> // Main CI workflow gz_software.each { gz_sw -> - def String ci_main_name = "ignition_${gz_sw/sim/gazebo}-ci-manual_any" + def String ci_main_name = "ignition_${software_name}-ci-manual_any" def gz_ci_main = pipelineJob(ci_main_name) OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) } diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index a2df6a408..59f7b995d 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -1,12 +1,12 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -// IGNITION COLLECTIONS +// GZ COLLECTIONS arch = 'amd64' -ignition_nightly = 'garden' +gz_nightly = 'garden' -ignition_collections = [ +gz_collections = [ [ name : 'citadel', distros : [ 'bionic' ], ], @@ -39,7 +39,7 @@ ignition_collections = [ ], ] -ignition_collection_jobs = +gz_collection_jobs = [ 'citadel' : [ 'ign_common-ign-3-win', @@ -292,13 +292,14 @@ ignition_collection_jobs = def DISABLE_TESTS = false // Testing compilation from source -ignition_collections.each { gz_collection -> +gz_collections.each { gz_collection -> // COLCON - Windows gz_collection_name = gz_collection.get('name') - def ignition_win_ci_job = job("ign_${gz_collection_name/sim/gazebo}-ci-win") + + def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") Globals.gazebodistro_branch = true - OSRFWinCompilation.create(ignition_win_ci_job, false) - ignition_win_ci_job.with + OSRFWinCompilation.create(gz_win_ci_job, false) + gz_win_ci_job.with { steps { batchFile("""\ @@ -312,7 +313,7 @@ ignition_collections.each { gz_collection -> gz_collection.get('distros').each { distro -> // INSTALL JOBS: // -------------------------------------------------------------- - def install_default_job = job("ignition_${gz_collection_name/sim/gazebo}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_collection_name}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) install_default_job.with @@ -321,7 +322,7 @@ ignition_collections.each { gz_collection -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "ignition-${gz_collection_name/sim/gazebo}" + def dev_package = "ignition-${gz_collection_name}" label "gpu-reliable" @@ -343,35 +344,35 @@ ignition_collections.each { gz_collection -> // MAC Brew CI job // -------------------------------------------------------------- - def gz_brew_ci_job = job("ignition_${gz_collection_name/sim/gazebo}-ci-main-homebrew-amd64") + def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/ign-${gz_collection_name/sim/gazebo}", + "gazebosim/ign-${gz_collection_name}", "main", - "ign-${gz_collection_name/sim/gazebo}") + "ign-${gz_collection_name}") gz_brew_ci_job.with { steps { shell("""\ #!/bin/bash -xe - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${gz_collection_name/sim/gazebo}" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${gz_collection_name}" """.stripIndent()) } } // MAC Brew bottle install job // -------------------------------------------------------------- - def ignition_brew_install_bottle_job = job("ignition_${gz_collection_name/sim/gazebo}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(ignition_brew_install_bottle_job) + def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") + OSRFBrewInstall.create(gz_brew_install_bottle_job) - ignition_brew_install_bottle_job.with + gz_brew_install_bottle_job.with { triggers { cron('@daily') } - def bottle_name = "ignition-${gz_collection_name/sim/gazebo}" + def bottle_name = "ignition-${gz_collection_name}" steps { shell("""\ @@ -395,7 +396,7 @@ ignition_collections.each { gz_collection -> // DEBBUILD: linux package builder // -------------------------------------------------------------- - def build_pkg_job = job("ign-${gz_collection_name/sim/gazebo}-debbuilder") + def build_pkg_job = job("ign-${gz_collection_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with { @@ -412,12 +413,12 @@ ignition_collections.each { gz_collection -> dashboardView("ign-${gz_collection_name}") { jobs { - ignition_collection_jobs["${gz_collection_name}"].each { jobname -> + gz_collection_jobs["${gz_collection_name}"].each { jobname -> name(jobname) } - if (gz_collection_name == ignition_nightly) { + if (gz_collection_name == gz_nightly) { // add nightly debbuild jobs too - ignition_collections.find { it.get('name') == ignition_nightly }.get('nightly_jobs').each { job -> + gz_collections.find { it.get('name') == gz_nightly }.get('nightly_jobs').each { job -> name(job.getValue().get('debbuild') + '-debbuilder') } } @@ -471,7 +472,7 @@ def get_nightly_branch(collection_data, ign_package) collection_data = [] list_of_pkgs = "" -collection_data = ignition_collections.find { it.get('name') == ignition_nightly } +collection_data = gz_collections.find { it.get('name') == gz_nightly } collection_data = collection_data.get('nightly_jobs') collection_data.each { job -> @@ -479,7 +480,7 @@ collection_data.each { job -> list_of_pkgs = "${list_of_pkgs} ${debbuild}" } -def nightly_scheduler_job = job("ignition-${ignition_nightly}-nightly-scheduler") +def nightly_scheduler_job = job("ignition-${gz_nightly}-nightly-scheduler") OSRFUNIXBase.create(nightly_scheduler_job) nightly_scheduler_job.with From 4c0b7ebaa48fece24e870533017fe94f34e21851 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 Aug 2022 21:47:13 +0200 Subject: [PATCH 06/17] Fix var name --- jenkins-scripts/dsl/ignition_collection.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index f9e18369e..7010d27bb 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -347,7 +347,7 @@ gz_collections.each { gz_collection -> def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/gz-${ign_collection_name}", + "gazebosim/gz-${gz_collection_name}", "main", "ign-${gz_collection_name}") gz_brew_ci_job.with From 23a2e96bad81dd290d3197012d5effebdf7fa1f6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 Aug 2022 22:10:41 +0200 Subject: [PATCH 07/17] Fix support to keep ign branches --- jenkins-scripts/dsl/ignition.dsl | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 6390a270d..a4eb246b3 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -43,7 +43,7 @@ gz_no_test = [ ] // testing and debbuild job. // No branches in gz_branches means no released branches (only CI on // main, ABI check, install pkg) -gz_branches = [ 'cmake' : [ '2' ], +ignition_branches = [ 'cmake' : [ '2' ], 'common' : [ '1', '3', '4' ], 'fuel-tools' : [ '1', '4', '7' ], 'gazebo' : [ '3', '6' ], @@ -155,6 +155,18 @@ Map merge_maps(Map[] sources) { } } +// return major versions supported or empty if just 0,1 series under +// -dev package. +ArrayList supported_ign_branches(String ign_software) +{ + major_versions_registered = ignition_branches["${ign_software}"] + + if (major_versions_registered == null) + return [ '' ] + + return major_versions_registered +} + // return major versions supported or empty if just 0,1 series under // -dev package. ArrayList supported_gz_branches(String gz_software) @@ -179,21 +191,21 @@ ArrayList prerelease_branches(String gz_software) } // return all ci branch names -ArrayList all_branches(String gz_software) +ArrayList all_branches(String software_name) { List branches = new ArrayList(); - supported_gz_branches("${gz_software}").each { major_version -> + supported_ign_branches("${software_name}").each { major_version -> if ("${major_version}") { - branches.add("ign-${gz_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } - supported_gz_branches("${gz_software}").each { major_version -> + supported_gz_branches("${software_name}").each { major_version -> if ("${major_version}") { - branches.add("gz-${gz_software}${major_version}") + branches.add("gz-${software_name}${major_version}") } } branches.add('main') - prerelease_branches("${gz_software}").each { branch -> + prerelease_branches("${software_name}").each { branch -> if ("${branch}") { branches.add(branch) } From 3f8ea64df6573db7bf3fe0066a6c7d8104f8a96b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 Aug 2022 23:56:22 +0200 Subject: [PATCH 08/17] Fix spacing and return brew trick --- jenkins-scripts/dsl/ignition.dsl | 47 ++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index a4eb246b3..b5dbf4ab5 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -3,25 +3,25 @@ import javaposse.jobdsl.dsl.Job // IGNITION PACKAGES gz_software = [ 'cmake', - 'common', - 'fuel-tools', - 'gazebo', - 'gui', - 'launch', - 'math', - 'msgs', - 'physics', - 'plugin', - 'rendering', - 'sensors', - 'tools', - 'transport', - 'utils' ] + 'common', + 'fuel-tools', + 'gazebo', + 'gui', + 'launch', + 'math', + 'msgs', + 'physics', + 'plugin', + 'rendering', + 'sensors', + 'tools', + 'transport', + 'utils' ] // DESC: need gpu/display for tests gz_gpu = [ 'gazebo', - 'gui', - 'rendering', - 'sensors' ] + 'gui', + 'rendering', + 'sensors' ] // DESC: software does not support cmake warnings enabled gz_no_cmake_warnings = [ 'cmake', 'common', @@ -58,6 +58,7 @@ ignition_branches = [ 'cmake' : [ '2' ], 'tools' : [ '1' ], 'transport' : [ '4', '8', '11' ], 'utils' : [ '1' ]] + gz_branches = [ 'cmake' : [ '3' ], 'common' : [ '5' ], 'fuel-tools' : [ '8' ], @@ -81,13 +82,13 @@ gz_prerelease_branches = [] // don't appear in gz_branches (like nightly builders or 0-debbuild // jobs for the special cases of foo0 packages) gz_extra_debbuild = [ 'gazebo7', - 'utils1'] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 + 'utils1' ] // see comment https://github.com/gazebo-tooling/release-tools/pull/431#issuecomment-815099918 // DESC: exclude ignition from generate any install testing job gz_no_pkg_yet = [ ] // DESC: major versions that has a package in the prerelease repo. Should // not appear in gz_no_pkg_yet nor in gz_branches gz_prerelease_pkgs = [ 'placeholder' : [ - '1': [ 'bionic' ]], + '1': [ 'bionic' ]], ] // packages using colcon for windows compilation while migrating all them to // this solution @@ -612,7 +613,9 @@ gz_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${software_name}" + software_name="gz-${ign_sw}" + [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) } @@ -645,7 +648,9 @@ gz_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "${software_name}" + software_name="gz-${ign_sw}" + [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) } From b78b57da476c869c8954731f153aa1ff9ba098a1 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 18 Aug 2022 23:58:45 +0200 Subject: [PATCH 09/17] Fix variable name --- jenkins-scripts/dsl/ignition.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index b5dbf4ab5..e26fc630b 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -613,8 +613,8 @@ gz_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) @@ -648,8 +648,8 @@ gz_software.each { gz_sw -> then /bin/bash -xe "\$HOMEBREW_SCRIPT" else - software_name="gz-${ign_sw}" - [[ ${ign_sw} == 'gazebo' ]] && software_name="gz-sim" + software_name="gz-${software_name}" + [[ ${software_name} == 'gazebo' ]] && software_name="gz-sim" /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "\${software_name}" fi """.stripIndent()) From 4dff9c4e65113acc33b9304da77cc9983c2d36f6 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 00:25:12 +0200 Subject: [PATCH 10/17] Fix Windows generation --- jenkins-scripts/dsl/ignition.dsl | 36 +++++++++++++++----------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index e26fc630b..5835ed122 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -516,7 +516,7 @@ gz_software.each { gz_sw -> supported_arches.each { arch -> // -------------------------------------------------------------- // branches CI job scm@daily - all_branches("${software_name}").each { branch -> + all_branches("${gz_sw}").each { branch -> // 1. Standard CI software_name = gz_sw // Necessary substitution. gz_sw won't overwrite @@ -702,24 +702,22 @@ gz_software.each { gz_sw -> // 1. any gz_software.each { gz_sw -> - if (is_a_colcon_package(software_name)) { + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal - gz_win_ci_any_job_name = "ign_${software_name}-pr-win" + gz_win_ci_any_job_name = "ign_${gz_sw}-pr-win" Globals.gazebodistro_branch = true } else { - gz_win_ci_any_job_name = "ignition_${software_name}-ci-pr_any-windows7-amd64" + gz_win_ci_any_job_name = "ignition_${gz_sw}-ci-pr_any-windows7-amd64" Globals.gazebodistro_branch = false } supported_branches = [] gz_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - ign_software_name = gz_sw // Necessary substitution. gz_sw won't overwrite // ign-gazebo only support Windows from ign-gazebo5 if (gz_sw == 'gazebo') supported_branches = [ 'ign-gazebo6', 'gz-sim7', 'main' ] - ign_software_name = "gazebo" gz_software_name = "sim" // ign-launch only support Windows from ign-launch5 @@ -728,16 +726,16 @@ gz_software.each { gz_sw -> def gz_win_ci_any_job = job(gz_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, - "gazebosim/gz-${software_name}", - enable_testing(software_name), + "gazebosim/gz-${gz_software_name}", + enable_testing(gz_software_name), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(software_name)) + enable_cmake_warnings(gz_software_name)) gz_win_ci_any_job.with { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${ign_software_name}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_software_name}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -746,25 +744,25 @@ gz_software.each { gz_sw -> ci_pr_any_list[gz_sw] << gz_win_ci_any_job_name // 2. main, release branches - all_branches("${software_name}").each { branch -> - if (is_a_colcon_package(software_name)) { + all_branches("${gz_sw}").each { branch -> + if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal if (branch == 'main') branch_name = "ci" else branch_name = branch - gz_sw - gz_win_ci_job_name = "ign_${ign_software_name}-${branch_name}-win" + gz_win_ci_job_name = "ign_${gz_software_name}-${branch_name}-win" } else { - gz_win_ci_job_name = "ignition_${ign_software_name}-ci-${branch}-windows7-amd64" + gz_win_ci_job_name = "ignition_${gz_software_name}-ci-${branch}-windows7-amd64" } def gz_win_ci_job = job(gz_win_ci_job_name) OSRFWinCompilation.create(gz_win_ci_job, - enable_testing(software_name), - enable_cmake_warnings(software_name)) + enable_testing(gz_sw), + enable_cmake_warnings(gz_sw)) OSRFGitHub.create(gz_win_ci_job, - "gazebosim/gz-${software_name}", + "gazebosim/gz-${gz_sw}", "${branch}") gz_win_ci_job.with @@ -783,7 +781,7 @@ gz_software.each { gz_sw -> steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${software_name}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } @@ -792,7 +790,7 @@ gz_software.each { gz_sw -> // Main CI workflow gz_software.each { gz_sw -> - def String ci_main_name = "ignition_${software_name}-ci-manual_any" + def String ci_main_name = "ignition_${gz_sw}-ci-manual_any" def gz_ci_main = pipelineJob(ci_main_name) OSRFCIWorkFlowMultiAnyGitHub.create(gz_ci_main, ci_pr_any_list[gz_sw]) } From 5f9f8343d6e4dce80c4e400ece576894f9518fe9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 00:48:15 +0200 Subject: [PATCH 11/17] Fix debbuilder creation --- jenkins-scripts/dsl/ignition.dsl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 5835ed122..44254499a 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -214,34 +214,34 @@ ArrayList all_branches(String software_name) return branches } -// + ArrayList all_debbuilders() { List branches = new ArrayList(); // add all supported branches - gz_software.each { gz_software -> - supported_gz_branches("${gz_software}").each { major_version -> + gz_software.each { software_name -> + supported_ign_branches("${software_name}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${gz_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } - supported_gz_branches("${gz_software}").each { major_version -> + supported_gz_branches("${software_name}").each { major_version -> if (major_version) { // No 1-debbuild versions, they use the unversioned job if ("${major_version}" == "0" || "${major_version}" == "1" ) major_version = "" - branches.add("ign-${gz_software}${major_version}") + branches.add("ign-${software_name}${major_version}") } } } // add all extra debbuilders gz_extra_debbuild.each { gz_name -> - branches.add("ign-${gz_name}") + branches.add("ign-${gz_name}_name") } return branches From b0c84e259970c441bc32cf7edbca714782c40c79 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 01:07:02 +0200 Subject: [PATCH 12/17] Fix Windows CI --- jenkins-scripts/dsl/ignition.dsl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 44254499a..dd7e74609 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -745,6 +745,7 @@ gz_software.each { gz_sw -> // 2. main, release branches all_branches("${gz_sw}").each { branch -> + println("BRANCH: ${gz_sw} - ${branch}") if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal @@ -752,9 +753,9 @@ gz_software.each { gz_sw -> branch_name = "ci" else branch_name = branch - gz_sw - gz_win_ci_job_name = "ign_${gz_software_name}-${branch_name}-win" + gz_win_ci_job_name = "ign_${gz_sw}-${branch_name}-win" } else { - gz_win_ci_job_name = "ignition_${gz_software_name}-ci-${branch}-windows7-amd64" + gz_win_ci_job_name = "ignition_${gz_sw}-ci-${branch}-windows7-amd64" } def gz_win_ci_job = job(gz_win_ci_job_name) From e0d8b9321e1a5b33a672bc423989188380e27af0 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 01:45:31 +0200 Subject: [PATCH 13/17] Fix install jobs --- jenkins-scripts/dsl/ignition.dsl | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index dd7e74609..a198c0cb3 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -259,7 +259,7 @@ Map supported_install_pkg_branches(String gz_software) // all_supported_distros map_of_stable_versions = [:] map_of_stable_versions[gz_software] = [:] - supported_gz_branches(gz_software).each { major_version -> + supported_ign_branches(gz_software).each { major_version -> new_relation = [:] new_relation[major_version] = all_supported_distros map_of_stable_versions[gz_software] << new_relation @@ -423,7 +423,7 @@ gz_software.each { gz_sw -> return supported_arches.each { arch -> - supported_install_pkg_branches(software_name).each { major_version, supported_distros -> + supported_install_pkg_branches(gz_sw).each { major_version, supported_distros -> supported_distros.each { distro -> extra_repos_str="" if ((gz_sw in gz_prerelease_pkgs) && @@ -436,15 +436,10 @@ gz_software.each { gz_sw -> if ("${major_version}" == "0" || "${major_version}" == "1") major_version = "" - software_name = gz_sw // Necessary substitution. gz_sw won't overwrite - - if (gz_sw == 'sim') - software_name = "gazebo" - // -------------------------------------------------------------- - def install_default_job = job("ignition_${software_name}${major_version}-install-pkg-${distro}-${arch}") + def install_default_job = job("ignition_${gz_sw}${major_version}-install-pkg-${distro}-${arch}") OSRFLinuxInstall.create(install_default_job) - include_gpu_label_if_needed(install_default_job, software_name) + include_gpu_label_if_needed(install_default_job, gz_sw) install_default_job.with { @@ -452,8 +447,8 @@ gz_software.each { gz_sw -> cron(Globals.CRON_EVERY_THREE_DAYS) } - def dev_package = "libignition-${software_name}${major_version}-dev" - def gzdev_project = "ignition-${software_name}${major_version}" + def dev_package = "libignition-${gz_sw}${major_version}-dev" + def gzdev_project = "ignition-${gz_sw}${major_version}" steps { shell("""\ @@ -557,7 +552,6 @@ all_debbuilders().each { debbuilder_name -> if (debbuilder_name.contains("-physics")) extra_str += '\nif [ $(uname -m) = "aarch64" ]; then export MAKE_JOBS=1; fi' - println("Generating: ${debbuilder_name}-debbuilder") def build_pkg_job = job("${debbuilder_name}-debbuilder") OSRFLinuxBuildPkg.create(build_pkg_job) build_pkg_job.with @@ -745,7 +739,6 @@ gz_software.each { gz_sw -> // 2. main, release branches all_branches("${gz_sw}").each { branch -> - println("BRANCH: ${gz_sw} - ${branch}") if (is_a_colcon_package(gz_sw)) { // colcon uses long paths and windows has a hard limit of 260 chars. Keep // names minimal From 94ec5a329a4179b9dd2def87349607ef822bb798 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 17:28:28 +0200 Subject: [PATCH 14/17] Fix problem with var name --- jenkins-scripts/dsl/ignition.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index a198c0cb3..65546bdd5 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -241,7 +241,7 @@ ArrayList all_debbuilders() } // add all extra debbuilders gz_extra_debbuild.each { gz_name -> - branches.add("ign-${gz_name}_name") + branches.add("ign-${gz_name}") } return branches From 3fdaea768f538b2f4445d325e58dedcd35d0a1ef Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 18:01:13 +0200 Subject: [PATCH 15/17] Use gz_sw variable instead of software name --- jenkins-scripts/dsl/ignition.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 65546bdd5..9323b33e8 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -720,16 +720,16 @@ gz_software.each { gz_sw -> def gz_win_ci_any_job = job(gz_win_ci_any_job_name) OSRFWinCompilationAnyGitHub.create(gz_win_ci_any_job, - "gazebosim/gz-${gz_software_name}", - enable_testing(gz_software_name), + "gazebosim/gz-${gz_sw}", + enable_testing(gz_sw), supported_branches, ENABLE_GITHUB_PR_INTEGRATION, - enable_cmake_warnings(gz_software_name)) + enable_cmake_warnings(gz_sw)) gz_win_ci_any_job.with { steps { batchFile("""\ - call "./scripts/jenkins-scripts/ign_${gz_software_name}-default-devel-windows-amd64.bat" + call "./scripts/jenkins-scripts/ign_${gz_sw}-default-devel-windows-amd64.bat" """.stripIndent()) } } From 662c8571604501140df82e48f7b31f7751554978 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 18:57:20 +0200 Subject: [PATCH 16/17] Restore gz- prefix for collection_name --- jenkins-scripts/dsl/ignition_collection.dsl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index 7010d27bb..f4c3266aa 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -356,7 +356,8 @@ gz_collections.each { gz_collection -> shell("""\ #!/bin/bash -xe - /bin/bash -xe "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "ignition-${gz_collection_name}" + /bin/bash -xe + "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" """.stripIndent()) } } From 825bcaa9cbb8d1e7e1aa29664dbe67d32fecb29a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Aug 2022 19:52:45 +0200 Subject: [PATCH 17/17] Update source-repo-scripts/bump_dependency.bash Co-authored-by: methylDragon --- source-repo-scripts/bump_dependency.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source-repo-scripts/bump_dependency.bash b/source-repo-scripts/bump_dependency.bash index e8d55d9b2..671b4f8f9 100755 --- a/source-repo-scripts/bump_dependency.bash +++ b/source-repo-scripts/bump_dependency.bash @@ -475,7 +475,7 @@ for ((i = 0; i < "${#LIBRARIES[@]}"; i++)); do find . -type f ! -name 'Changelog.md' ! -name 'Migration.md' -print0 | xargs -0 sed -i "s ${DEP_LIB}${DEP_PREV_VER} ${DEP_LIB}${DEP_VER} g" # Replace collection yaml branch names with main - if [[ s"${LIB}" == "gz-${COLLECTION}" ]]; then + if [[ "${LIB}" == "gz-${COLLECTION}" ]]; then find . -type f -name "collection-${COLLECTION}.yaml" -print0 | xargs -0 sed -i "s gz-${DEP_LIB}${DEP_VER} main g" fi