From f5e36f52db20dba712efd2528c00360d32691023 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 7 Feb 2025 17:30:22 +0100 Subject: [PATCH] WIP yum_repo_status: generate a table showing how much outdated 8.3 is from xs8 FIXME: - include successive xs8 update waves - migrate infos from Sam's wiki page to package_status.csv - show binrpms? - include "current" el7 versions - link proprietary packages replaced by ours - should take import_reason from provenance.csv as tooltip? - include upstream releases from GH? --- scripts/rpmwatcher/package_status.csv | 60 +++++++ scripts/rpmwatcher/repoquery.py | 94 +++++++++-- scripts/rpmwatcher/style.css | 52 ++++++ scripts/rpmwatcher/yum_repo_status | 222 ++++++++++++++++++++++++++ 4 files changed, 416 insertions(+), 12 deletions(-) create mode 100644 scripts/rpmwatcher/package_status.csv create mode 100644 scripts/rpmwatcher/style.css create mode 100755 scripts/rpmwatcher/yum_repo_status diff --git a/scripts/rpmwatcher/package_status.csv b/scripts/rpmwatcher/package_status.csv new file mode 100644 index 0000000..5782bb3 --- /dev/null +++ b/scripts/rpmwatcher/package_status.csv @@ -0,0 +1,60 @@ +SRPM_name;status;comment +auto-cert-kit;ignored;unused, why? +automake16;ignored;unused, why? +bpftool;ignored;unused, why? +capstone;ignored;unused, why? +citrix-crypto-module;ignored;proprietary, patched FIPS openssl +compiler-rt18;ignored;unused, why? +dlm;ignored;previous dependency for corosync +emu-manager;ignored;proprietary, replaced by xcp-emu-manager +epel-release;ignored;unused, why? +forkexecd;ignored;now in xapi +fuse;;breq for e2fsprogs 1.47 +gfs2-utils;ignored;unsupported fs +glib2;ignored;same version as el7, why? +golang;ignored;for newer xe-guest-utilities +hcp_nss;ignored;unused, “enforce any permitted user login as root” +hwloc;ignored;unused, why? +libbpf;ignored;unused, why? +libcgroup;ignored;unused, same version as el7, why? +libhbalinux;;unused? el7 fork, “Fix crash in fcoeadm/elxhbamgr on certain machines” +libnbd;ignored;unused, dep for xapi-storage-plugins +linuxconsoletools;ignored;unused, same version as el7, why? +mbootpack;;for secureboot? +message-switch;ignored;now in xapi +mpdecimal;ignored;unused, why? +ninja-build;ignored;unused +pbis-open;ignored;unused, likely linked to upgrade-pbis-to-winbind +pbis-open-upgrade;ignored;unused, likely linked to upgrade-pbis-to-winbind +pvsproxy;ignored;proprietary +python-monotonic;ignored;previous dependency for sm +python-tqdm;ignored;unused, dependency for pvsproxy +rrdd-plugins;ignored;now in xapi +ruby;ignored;unused, why? +sbd;;unused? "storage-based death functionality" +sm-cli;ignored;now in xapi +secureboot-certificates;ignored;proprietary, needs alternative? +security-tools;ignored;proprietary, pool_secret tool +sm-transport-lib;ignored;proprietary +squeezed;ignored;now in xapi +tix;ignored;unused, why? +upgrade-pbis-to-winbind;ignored;proprietary +v6d;ignored;proprietary, replaced by xcp-featured +varstored-guard;ignored;now in xapi +vendor-update-keys;ignored;proprietary +vgpu;ignored;proprietary +vhd-tool;ignored;now in xapi +wsproxy;ignored;now in xapi +xapi-clusterd;ignored;proprietary +xapi-nbd;ignored;now in xapi +xapi-storage;ignored;now in xapi +xapi-storage-plugins;ignored;proprietarized, forked as xcp-ng-xapi-storage +xapi-storage-script;ignored;now in xapi +xcp-networkd;ignored;now in xapi +xcp-rrdd;ignored;now in xapi +xencert;;"automated testkit for certifying storage hardware with XenServer" +xenopsd;ignored;now in xapi +xenserver-release;forked;xcp-ng-release +xenserver-snmp-agent;ignored;proprietary, SNMP MIB +xenserver-telemetry;ignored;proprietary, xapi plugin +xs-clipboardd;ignored;proprietary, replaced by xcp-clipboardd diff --git a/scripts/rpmwatcher/repoquery.py b/scripts/rpmwatcher/repoquery.py index f1667cb..de650c3 100644 --- a/scripts/rpmwatcher/repoquery.py +++ b/scripts/rpmwatcher/repoquery.py @@ -12,22 +12,59 @@ skip_if_unavailable=False """ +XCPNG_YUMREPO_USER_TMPL = """ +[xcpng-{section}{suffix}] +name=xcpng - {section}{suffix} +baseurl=https://koji.xcp-ng.org/repos/user/8/{version}/{section}/{rpmarch}/ +gpgkey=https://xcp-ng.org/RPM-GPG-KEY-xcpng +failovermethod=priority +skip_if_unavailable=False +""" + def setup_xcpng_yum_repos(*, yum_repo_d, sections, bin_arch, version): with open(os.path.join(yum_repo_d, "xcpng.repo"), "w") as yumrepoconf: for section in sections: + # HACK: use USER_TMPL if section ends with a number + if section[-1].isdigit(): + tmpl = XCPNG_YUMREPO_USER_TMPL + else: + tmpl = XCPNG_YUMREPO_TMPL + # binaries - block = XCPNG_YUMREPO_TMPL.format(rpmarch=bin_arch, - section=section, - version=version, - suffix='', - ) - yumrepoconf.write(block) + if bin_arch: + block = tmpl.format(rpmarch=bin_arch, + section=section, + version=version, + suffix='', + ) + yumrepoconf.write(block) # sources - block = XCPNG_YUMREPO_TMPL.format(rpmarch='Source', - section=section, - version=version, - suffix='-src', - ) + block = tmpl.format(rpmarch='Source', + section=section, + version=version, + suffix='-src', + ) + yumrepoconf.write(block) + + +XS8_YUMREPO_TMPL = """ +[xs8-{section}] +name=XS8 - {section} +baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}/ +failovermethod=priority +skip_if_unavailable=False + +[xs8-{section}-src] +name=XS8 - {section} source +baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}-source/ +failovermethod=priority +skip_if_unavailable=False +""" + +def setup_xs8_yum_repos(*, yum_repo_d, sections): + with open(os.path.join(yum_repo_d, "xs8.repo"), "w") as yumrepoconf: + for section in sections: + block = XS8_YUMREPO_TMPL.format(section=section) yumrepoconf.write(block) DNF_BASE_CMD = None @@ -47,7 +84,7 @@ def run_repoquery(args): cmd = DNF_BASE_CMD + ['repoquery'] + args logging.debug('$ %s', ' '.join(cmd)) ret = subprocess.check_output(cmd, universal_newlines=True).strip().split() - logging.debug('> %s', ret) + logging.debug('> %s', '\n'.join(ret)) return ret SRPM_BINRPMS_CACHE = {} # binrpm-nevr -> srpm-nevr @@ -141,3 +178,36 @@ def is_pristine_upstream(rpmname): if re.search(UPSTREAM_REGEX, rpmname): return True return False + +def rpm_parse_nevr(nevr, suffix): + "Parse into (name, epoch:version, release) stripping suffix from release" + m = re.match(RPM_NVR_SPLIT_REGEX, nevr) + assert m, f"{nevr} does not match NEVR pattern" + n, ev, r = m.groups() + if ":" in ev: + e, v = ev.split(":") + else: + e, v = "0", ev + if r.endswith(suffix): + r = r[:-len(suffix)] + return (n, e, v, r) + +def all_binrpms(): + args = [ + '--disablerepo=*-src', + '--qf=%{name}-%{evr}', # to avoid getting the arch + '--latest-limit=1', # only most recent for each package + '*', + ] + ret = set(run_repoquery(args)) + return ret + +def all_srpms(): + args = [ + '--disablerepo=*', '--enablerepo=*-src', + '--qf=%{name}-%{evr}', # to avoid getting the arch + '--latest-limit=1', # only most recent for each package + '*', + ] + ret = set(run_repoquery(args)) + return ret diff --git a/scripts/rpmwatcher/style.css b/scripts/rpmwatcher/style.css new file mode 100644 index 0000000..b309892 --- /dev/null +++ b/scripts/rpmwatcher/style.css @@ -0,0 +1,52 @@ +table { + border-collapse: collapse; + border-spacing: 0; +} +td { + padding: 2px 5px; + /* background-color: lightblue; */ +} + +tr.header { + vertical-align: top; +} +tr.ignored { + color: #888888; + background-color: #cccccc; +} +tr.notused { /* still to check */ + color: red; + background-color: #cccccc; +} + +th.xs { + background-color: #ddddff; +} +th.xcp { + background-color: #ddffdd; +} + +.nosource { + color: red; + background-color: black; +} + +.unexpected { + color: red; +} + +.outdated { + background-color: red; +} + +.uptodate { + background-color: lightgreen; +} + +.upstream { + background-color: #88ffdd; +} + +.better { + background-color: #77cc00; +} diff --git a/scripts/rpmwatcher/yum_repo_status b/scripts/rpmwatcher/yum_repo_status new file mode 100755 index 0000000..f68ca02 --- /dev/null +++ b/scripts/rpmwatcher/yum_repo_status @@ -0,0 +1,222 @@ +#! /usr/bin/env python3 + +from collections import namedtuple, OrderedDict +import csv +import logging +import os +import rpm +import sys +import tempfile + +import repoquery + +ARCH = "x86_64" +XCP_VERSION = "8.3" +FILTER_UPSTREAM = False + +def evr_format(evr): + if evr == "=": # dirty special case + return evr + if evr is None: + return "-" + return (f"{evr[0]}:" if evr[0] != "0" else "") + "-".join(evr[1:]) + +# Filters an iterator of (n, e, v, r) for newest evr of each `n`. +# Older versions are allowed to appear before the newer ones. +def filter_best_evr(nevrs): + best = {} + for (n, e, v, r) in nevrs: + if n not in best or rpm.labelCompare(best[n], (e, v, r)) < 0: + best[n] = (e, v, r) + yield (n, e, v, r) + # else (e, v, r) is older than a previously-seen version, drop + +def main(): + logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO) + + # collect data from repos + + # ... xcp-ng + + xcp_sets = OrderedDict() + for (label, sections) in (("released", ['base', 'updates']), + ("testing", ["testing"]), + ("ci", ["ci"]), + ("incoming", ["incoming"]), + ("ydi1", ["ydi1"]), + ("dtt1", ["dtt1"]), + ): + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xcpng_yum_repos(yum_repo_d=yumrepod, + sections=sections, + bin_arch=None, + version=XCP_VERSION) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + #repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XCP-ng %s SRPMs", label) + xcp_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xcp_sets[label] = { + n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xcpng{XCP_VERSION}") + for nevr in xcp_srpms)} + + logging.info(f"{label}: {len(xcp_sets[label])}") + + # ... xs8 + + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod, + sections=['base', 'normal'], + ) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + logging.debug("fill cache with XS info") + repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XS SRPMs") + xs8_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + xs8_rpms_sources = {nevr for nevr in repoquery.SRPM_BINRPMS_CACHE.keys() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xs8_srpms_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8_srpms)} + xs8_rpms_sources_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8_rpms_sources)} + + logging.info(f"xs8 src: {len(xs8_srpms_set)}") + logging.info(f"xs8 bin: {len(xs8_rpms_sources_set)}") + + # ... xs8 earlyaccess + + with (tempfile.NamedTemporaryFile() as dnfconf, + tempfile.TemporaryDirectory() as yumrepod): + + repoquery.setup_xs8_yum_repos(yum_repo_d=yumrepod, + sections=['earlyaccess'], + ) + repoquery.dnf_setup(dnf_conf=dnfconf.name, yum_repo_d=yumrepod) + logging.debug("fill cache with XS EA info") + repoquery.fill_srpm_binrpms_cache() + + logging.debug("get all XS EA SRPMs") + xs8ea_srpms = {nevr for nevr in repoquery.all_srpms() + if not FILTER_UPSTREAM or not repoquery.is_pristine_upstream(nevr)} + + xs8ea_srpms_set = {n: (e, v, r) + for (n, e, v, r) + in filter_best_evr(repoquery.rpm_parse_nevr(nevr, f".xs8") + for nevr in xs8ea_srpms)} + + logging.info(f"xs8 EA src: {len(xs8ea_srpms_set)}") + + # ... collect + + all_srpms = set(xs8_srpms_set.keys()) | xs8_rpms_sources_set.keys() | xs8ea_srpms_set.keys() + for label, srpms in xcp_sets.items(): + all_srpms |= srpms.keys() + + logging.info(f"all: {len(all_srpms)}") + + # extra metadata + + with open('package_status.csv', newline='') as csvfile: + csvreader = csv.reader(csvfile, delimiter=';', quotechar='|') + headers = next(csvreader) + assert headers == ["SRPM_name", "status", "comment"], f"unexpected headers {headers!r}" + PackageStatus = namedtuple("PackageStatus", headers[1:]) + PACKAGE_STATUS={row[0]: PackageStatus(*row[1:]) + for row in csvreader} + + # output + + with open(f"repo-status-{XCP_VERSION}.html", "w") as outfile: + print('', file=outfile) + print('', file=outfile) + print('', file=outfile) + print('', file=outfile) + print('', file=outfile) + + for srpm in sorted(all_srpms): + xs_ver_src = xs8_srpms_set.get(srpm, None) + xsea_ver_src = xs8ea_srpms_set.get(srpm, None) + if xs_ver_src and xsea_ver_src == xs_ver_src: + xsea_ver_src = "=" # don't overload with useless info # FIXME + xs_ver_bin = xs8_rpms_sources_set.get(srpm, None) + xcp_vers = {label: xcp_sets[label].get(srpm, None) + for label in xcp_sets} + + srpm_status = PACKAGE_STATUS.get(srpm, None) + + if srpm_status: + tooltip = srpm_status.comment + else: + tooltip = "" + + if srpm_status and srpm_status.status == "ignored": + row_classes = "ignored" + elif all(ver is None for ver in xcp_vers.values()) and xs_ver_bin is None: + row_classes = "notused" + else: + row_classes = "" + + # XS source + xss_classes = "xs" + if srpm in xs8_rpms_sources_set and srpm not in xs8_srpms_set: + xss_classes += " nosource" + + # XS binary + xsb_classes = "xs" + if xs_ver_bin and xs_ver_src != xs_ver_bin: + xsb_classes += " nosource" + + print(f'', file=outfile) + print('
package', file=outfile) + print(f'XenServer 8XCP-ng 8.3
srcrpmbinrpmEA srcrpm', file=outfile) + for label in xcp_sets: + print(f'{label}', file=outfile) + print(f'
{srpm}' + f'{evr_format(xs_ver_src)}' + f'{evr_format(xs_ver_bin)}', + f'{evr_format(xsea_ver_src)}', + file=outfile) + + # XCP-ng + ref_xs_ver = xs_ver_bin if xs_ver_bin else xs_ver_src + for label in xcp_sets: + xcp_ver = xcp_vers[label] + xcp_ver_str = evr_format(xcp_ver) + classes = "xcp" + if repoquery.is_pristine_upstream(xcp_ver_str): + classes += " upstream" + if not xcp_ver: + pass + elif xcp_ver and xcp_ver == ref_xs_ver: + classes += " uptodate" + elif not ref_xs_ver: + classes += " better" + elif not ref_xs_ver: + pass + elif rpm.labelCompare(xcp_ver, ref_xs_ver) < 0: + classes += " outdated" + else: # xcp_ver > ref_xs_ver + classes += " better" + if row_classes == "notused" and xcp_ver: + classes += " unexpected" + + print(f'{xcp_ver_str}', file=outfile) + print(f'
', file=outfile) + +if __name__ == "__main__": + sys.exit(main())