Merge branch 'master' into cottsay/stdeb-dependencies

ros_buildfarm/jenkins.py

@@ -274,9 +274,8 @@ def _diff_configs(remote_config, new_config, context_lines):
     if ElementTree.tostring(remote_root) == ElementTree.tostring(new_root):
         return []
 
-    encoding_type = 'utf-8' if sys.version_info[0] < 3 else 'unicode'
-    xml1 = ElementTree.tostring(remote_root, encoding=encoding_type)
-    xml2 = ElementTree.tostring(new_root, encoding=encoding_type)
+    xml1 = ElementTree.tostring(remote_root, encoding='unicode')
+    xml2 = ElementTree.tostring(new_root, encoding='unicode')
     lines1 = xml1.splitlines()
     lines2 = xml2.splitlines()
 

ros_buildfarm/scripts/release/generate_release_script.py

@@ -42,10 +42,18 @@ def main(argv=sys.argv[1:]):
     add_argument_os_name(parser)
     add_argument_os_code_name(parser)
     add_argument_arch(parser)
+    parser.add_argument(
+        '--skip-binary',
+        action='store_true',
+        help='Skip the entire binary package build process')
     parser.add_argument(
         '--skip-install',
         action='store_true',
         help='Skip trying to install binarydeb')
+    parser.add_argument(
+        '--skip-source',
+        action='store_true',
+        help='Skip the entire source package build process')
     args = parser.parse_args(argv)
 
     package_format = package_format_mapping[args.os_name]
@@ -103,28 +111,29 @@ def beforeInclude(self, *args, **kwargs):
         args.package_name, args.os_name, args.os_code_name, args.arch)
 
     separator_index = hook.scripts.index('--')
-    source_scripts = hook.scripts[:separator_index]
-    binary_scripts = hook.scripts[separator_index + 1:]
-
-    # inject additional argument to skip fetching sourcedeb from repo
-    script_name = '/run_binary%s_job.py ' % deb_or_pkg
-    additional_argument = '--skip-download-sourcepkg '
-    for i, script in enumerate(binary_scripts):
-        offset = script.find(script_name)
-        if offset != -1:
-            offset += len(script_name)
-            script = script[:offset] + additional_argument + script[offset:]
-            binary_scripts[i] = script
-            break
-
-    # remove rm command for sourcedeb location
-    rm_command = 'rm -fr $WORKSPACE/binary%s' % deb_or_pkg
-    for i, script in enumerate(binary_scripts):
-        offset = script.find(rm_command)
-        if offset != -1:
-            script = script[:offset] + script[offset + len(rm_command):]
-            binary_scripts[i] = script
-            break
+    source_scripts = [] if args.skip_source else hook.scripts[:separator_index]
+    binary_scripts = [] if args.skip_binary else hook.scripts[separator_index + 1:]
+
+    if source_scripts:
+        # inject additional argument to skip fetching sourcedeb from repo
+        script_name = '/run_binary%s_job.py ' % deb_or_pkg
+        additional_argument = '--skip-download-sourcepkg '
+        for i, script in enumerate(binary_scripts):
+            offset = script.find(script_name)
+            if offset != -1:
+                offset += len(script_name)
+                script = script[:offset] + additional_argument + script[offset:]
+                binary_scripts[i] = script
+                break
+
+        # remove rm command for sourcedeb location
+        rm_command = 'rm -fr $WORKSPACE/binary%s' % deb_or_pkg
+        for i, script in enumerate(binary_scripts):
+            offset = script.find(rm_command)
+            if offset != -1:
+                script = script[:offset] + script[offset + len(rm_command):]
+                binary_scripts[i] = script
+                break
 
     if args.skip_install:
         # remove install step

ros_buildfarm/status_page.py

@@ -582,16 +582,15 @@ def _version_is_gt_other(version, other_version):
 def _get_comparable_loose_versions(version_str1, version_str2):
     loose_version1 = LooseVersion(version_str1)
     loose_version2 = LooseVersion(version_str2)
-    if sys.version_info[0] > 2:
-        # might raise TypeError in Python 3: http://bugs.python.org/issue14894
-        version_parts1 = loose_version1.version
-        version_parts2 = loose_version2.version
-        for i in range(min(len(version_parts1), len(version_parts2))):
-            try:
-                version_parts1[i] < version_parts2[i]
-            except TypeError:
-                version_parts1[i] = str(version_parts1[i])
-                version_parts2[i] = str(version_parts2[i])
+    # might raise TypeError in Python 3: http://bugs.python.org/issue14894
+    version_parts1 = loose_version1.version
+    version_parts2 = loose_version2.version
+    for i in range(min(len(version_parts1), len(version_parts2))):
+        try:
+            version_parts1[i] < version_parts2[i]
+        except TypeError:
+            version_parts1[i] = str(version_parts1[i])
+            version_parts2[i] = str(version_parts2[i])
     return loose_version1, loose_version2
 
 

ros_buildfarm/templates/release/deb/binarypkg_job.xml.em

@@ -119,7 +119,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Build Dockerfile - binarydeb task"',
         'cd $WORKSPACE/docker_generating_docker',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+        'docker build --network=host --force-rm -t binarydeb_task_generation.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - binarydeb task"',
@@ -138,6 +138,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         ' --rm ' +
         ' --cidfile=$WORKSPACE/docker_generating_docker/docker.cid' +
         ' -e=TRAVIS=$TRAVIS' +
+        ' --net=host' +
         ' -e=ROS_BUILDFARM_PULL_REQUEST_BRANCH=$ROS_BUILDFARM_PULL_REQUEST_BRANCH' +
         ' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
         ' -v $WORKSPACE/binarydeb:/tmp/binarydeb' +
@@ -159,7 +160,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         '# build and run build_binarydeb Dockerfile',
         'cd $WORKSPACE/docker_build_binarydeb',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t binarydeb_build.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+        'docker build --network=host --force-rm -t binarydeb_build.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - build binarydeb"',
@@ -224,7 +225,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
 @#         'echo "# BEGIN SECTION: Build Dockerfile - install"',
 @#         'cd $WORKSPACE/docker_install_binarydeb',
 @#         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-@#         'docker build --force-rm -t binarydeb_install.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
+@#         'docker build --network=host --force-rm -t binarydeb_install.%s_%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, arch, pkg_name),
 @#         'echo "# END SECTION"',
 @#         '',
 @#         'echo "# BEGIN SECTION: Run Dockerfile - install"',

ros_buildfarm/templates/release/deb/sourcepkg_job.xml.em

@@ -104,7 +104,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Build Dockerfile - generate sourcedeb"',
         'cd $WORKSPACE/docker_sourcedeb',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t sourcedeb.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name),
+        'docker build --network=host --force-rm -t sourcedeb.%s_%s_%s_%s .' % (rosdistro_name, os_name, os_code_name, pkg_name),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generate sourcedeb"',

ros_buildfarm/templates/release/release_script.sh.em

@@ -45,6 +45,7 @@ echo "Using workspace for binary: $WORKSPACE"
 mkdir -p $WORKSPACE
 cd $WORKSPACE
 
+@[if source_scripts]@
 echo ""
 echo "Get artifacts from source job"
 @[if package_format == 'deb']@
@@ -56,6 +57,7 @@ mkdir -p $WORKSPACE/binarypkg/source
 @[else]@
 @{assert False, "Unsupported packaging format '%s'" % package_format}@
 @[end if]@
+@[end if]@
 
 @(TEMPLATE(
     'devel/devel_script_build.sh.em',

ros_buildfarm/templates/release/rpm/binarypkg_job.xml.em

@@ -130,6 +130,8 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
     ] + ([
         'if [ ! -d "$HOME/.ccache" ]; then mkdir $HOME/.ccache; fi',
     ] if shared_ccache else []) + [
+        '# If using Podman, change the user namespace to preserve UID. No effect if using Docker.',
+        'export PODMAN_USERNS=keep-id',
         'docker run' +
         ' --rm' +
         ' --privileged' +

ros_buildfarm/templates/release/rpm/binarypkg_task.Dockerfile.em

@@ -30,6 +30,7 @@ RUN dnf update --refresh -y
 RUN echo -e "@('\\n'.join(key.splitlines()))" > /etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i)
 @[end for]@
 COPY mock_config.cfg /etc/mock/ros_buildfarm.cfg
+RUN chmod 644 /etc/mock/ros_buildfarm.cfg
 
 USER buildfarm
 ENTRYPOINT ["sh", "-c"]

ros_buildfarm/templates/release/rpm/mock_config.cfg.em

@@ -1,5 +1,13 @@
 include('/etc/mock/default.cfg')
 
+def _expanded(k, opts):
+    """Get a config value and ensure it has been expanded."""
+    exp, opts['__jinja_expand'] = opts.get('__jinja_expand', False), True
+    try:
+        return opts[k]
+    finally:
+        opts['__jinja_expand'] = exp
+
 # Change the root name since we're modifying the chroot
 config_opts['root'] += '-ros-buildfarm'
 
@@ -10,7 +18,13 @@ config_opts['use_bootstrap'] = False
 config_opts['chroot_setup_cmd'] += ' python3-rpm-macros'
 
 # Install weak dependencies to get group members
-config_opts[f'dnf_builddep_opts'] = config_opts.get(f'dnf_builddep_opts', []) + ['--setopt=install_weak_deps=True']
+package_manager = _expanded('package_manager', config_opts)
+additional_opts = ['--setopt=install_weak_deps=True']
+config_opts[f'{package_manager}_builddep_opts'] = config_opts.get(f'{package_manager}_builddep_opts', []) + additional_opts
+
+# Deal with dnf -> dnf4 transition - see rpm-software-management/mock#1496
+if package_manager == 'dnf':
+    config_opts[f'dnf4_builddep_opts'] = config_opts.get(f'dnf4_builddep_opts', []) + additional_opts
 
 @[if env_vars]@
 # Set environment vars from the build config

ros_buildfarm/templates/release/rpm/sourcepkg_job.xml.em

@@ -110,6 +110,8 @@ but disabled since the package is blacklisted (or not whitelisted) in the config
         'echo "# BEGIN SECTION: Run Dockerfile - generate sourcerpm"',
         'rm -fr $WORKSPACE/sourcepkg',
         'mkdir -p $WORKSPACE/sourcepkg',
+        '# If using Podman, change the user namespace to preserve UID. No effect if using Docker.',
+        'export PODMAN_USERNS=keep-id',
         'docker run' +
         ' --rm' +
         ' --privileged' +

ros_buildfarm/templates/release/rpm/sourcepkg_task.Dockerfile.em

@@ -25,6 +25,7 @@ RUN dnf update --refresh -y
 RUN echo -e "@('\\n'.join(key.splitlines()))" > /etc/pki/mock/RPM-GPG-KEY-ros-buildfarm-@(i)
 @[end for]@
 COPY mock_config.cfg /etc/mock/ros_buildfarm.cfg
+RUN chmod 644 /etc/mock/ros_buildfarm.cfg
 
 USER buildfarm
 ENTRYPOINT ["sh", "-c"]

ros_buildfarm/templates/snippet/add_distribution_repositories.Dockerfile.em

@@ -14,10 +14,10 @@ RUN for i in 1 2 3; do apt-get update && apt-get install -q -y gnupg ca-certific
 @[for i, key in enumerate(distribution_repository_keys)]@
 RUN echo "@('\\n'.join(key.splitlines()))" > /tmp/keys/@(i).key@[if key] && apt-key add /tmp/keys/@(i).key@[end if]
 @[end for]@
-@[for url in distribution_repository_urls]@
-RUN echo deb @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list
+@[for i, url in enumerate(distribution_repository_urls)]@
+RUN echo deb @[if not distribution_repository_keys[i]][trusted=yes] @[end if]@ @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list
 @[if add_source and url == target_repository]@
-RUN echo deb-src @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list
+RUN echo deb-src @[if not distribution_repository_keys[i]][trusted=yes] @[end if]@ @url @os_code_name main | tee -a /etc/apt/sources.list.d/buildfarm.list
 @[end if]@
 @[end for]@
 @# On Ubuntu Trusty a newer version of dpkg is required to install Debian packages created by stdeb on newer distros

setup.py

@@ -25,6 +25,7 @@
     # - ros_buildfarm/__init__.py
     # - stdeb.cfg
     'version': '3.0.1-master',
+    'python_requires': '>=3.6',
     'packages': find_packages(exclude=['test']),
     'package_data': {
         'ros_buildfarm.templates': [
@@ -72,9 +73,6 @@
     'license': 'Apache 2.0',
 }
 
-if os.sys.version_info[0] == 2:
-    kwargs['install_requires'].append('configparser')
-
 if 'SKIP_PYTHON_MODULES' in os.environ:
     kwargs['packages'] = []
 elif 'SKIP_PYTHON_SCRIPTS' in os.environ:

stdeb.cfg

@@ -3,26 +3,18 @@ Debian-Version: 100
 ; ros-buildfarm-modules same version (without the branch suffix) as in:
 ; - ros_buildfarm/__init__.py
 ; - setup.py
-Depends: python-argparse, python-catkin-pkg-modules, python-ros-buildfarm-modules (>= 3.0.0), python-rosdistro-modules (>= 1.0.0), python-yaml
-; ros-buildfarm-modules same version (without the branch suffix) as in:
-; - ros_buildfarm/__init__.py
-; - setup.py
 Depends3: python3-catkin-pkg-modules (>= 0.2.6), python3-ros-buildfarm-modules (>= 3.0.0), python3-rosdistro-modules (>= 1.0.0), python3-yaml
-Conflicts: python3-ros-buildfarm
 Conflicts3: python-ros-buildfarm
-Suite: xenial yakkety zesty artful bionic cosmic disco eoan stretch buster
-Suite3: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster
-Python2-Depends-Name: python
+Suite3: focal jammy noble bookworm trixie
+No-Python2:
+X-Python3-Version: >= 3.6
 Setup-Env-Vars: SKIP_PYTHON_MODULES=1
 
 [ros_buildfarm_modules]
-Depends: python-catkin-pkg-modules, python-configparser, python-empy, python-rosdistro-modules (>= 1.0.0), python-yaml, python3-empy, python3-vcstool (>= 0.1.37)
 Depends3: python3-catkin-pkg-modules (>= 0.2.6), python3-empy (< 4), python3-jenkinsapi, python3-rosdistro-modules (>= 1.0.0), python3-vcstool (>= 0.1.37), python3-yaml
-Conflicts: python-ros-buildfarm (<< 1.3.0)
 Conflicts3: python3-ros-buildfarm (<< 1.3.0)
-Replaces: python-ros-buildfarm (<< 1.3.0)
 Replaces3: python3-ros-buildfarm (<< 1.3.0)
-Suite: xenial yakkety zesty artful bionic cosmic disco eoan stretch buster
-Suite3: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster
-Python2-Depends-Name: python
+Suite3: focal jammy noble bookworm trixie
+No-Python2:
+X-Python3-Version: >= 3.6
 Setup-Env-Vars: SKIP_PYTHON_SCRIPTS=1