From 1ac24ed72febedcc502ce6e9304d7f9f38c9c015 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 15:56:19 +0100 Subject: [PATCH 01/17] osbuildmonitor: add osbuild monitor test data This commit contains an abreviated version of an osbuild jsonseq output log. Most of the repetitive data from the curl and rpm stages got removed as they don't add (much) to the understanding. The data was generated via: ``` $ sudo python3 -m osbuild --libdir . --monitor JSONSeqMonitor \ --export image --output-directory /tmp/out \ ./test/data/manifests/fedora-boot.json > osbuild-status-output.json ``` --- .../testdata/osbuild-monitor-output.json | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 pkg/osbuildmonitor/testdata/osbuild-monitor-output.json diff --git a/pkg/osbuildmonitor/testdata/osbuild-monitor-output.json b/pkg/osbuildmonitor/testdata/osbuild-monitor-output.json new file mode 100644 index 0000000000..bf3ee10b02 --- /dev/null +++ b/pkg/osbuildmonitor/testdata/osbuild-monitor-output.json @@ -0,0 +1,154 @@ +{"message": "starting ./test/data/manifests/fedora-boot.json", "context": {"origin": "osbuild.main_cli", "pipeline": {"stage": {}}, "id": "ede22f789202b9017bd24089276f57f559018c593914632c80206be12f5b3bb0"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589337.2034357} +{"message": "Starting pipeline source org.osbuild.curl", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "source org.osbuild.curl", "id": "598849389c35f93efe2412446f5ca6919434417b9bcea040ea5f9203de81db2c", "stage": {}}, "id": "69816755441434713b7567970edfdd42d58193f163e1fdd506274d52246e87f2"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589337.204283} +{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/k/kpartx-0.9.5-2.fc39.x86_64.rpm\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "source org.osbuild.curl", "id": "598849389c35f93efe2412446f5ca6919434417b9bcea040ea5f9203de81db2c", "stage": {}}, "id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8252223} +{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/l/langpacks-fonts-en-4.0-9.fc39.noarch.rpm\n", "context": {"id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8256931} +{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/l/libfsverity-1.4-10.fc39.x86_64.rpm\n", "context": {"id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8260126} +{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-updates-released-20240508/Packages/l/linux-firmware-20240410-1.fc39.noarch.rpm\n", "context": {"id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589407.0240848} +{"message": "Finished pipeline org.osbuild.curl", "context": {"id": "69816755441434713b7567970edfdd42d58193f163e1fdd506274d52246e87f2"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1}, "timestamp": 1731589407.0243955} +{"message": "Starting pipeline build", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "build", "id": "32e87da44d9a519e89770723a33b7ecdd4ab85b872ae6ab8aaa94bdef9a275c7", "stage": {}}, "id": "0020bdf60135d4a03d8db333f66d40386278bf55b39fd06ed18839da11d98f96"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.0338647} +{"message": "Starting module org.osbuild.rpm", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "build", "id": "32e87da44d9a519e89770723a33b7ecdd4ab85b872ae6ab8aaa94bdef9a275c7", "stage": {"name": "org.osbuild.rpm", "id": "bf00d0e1e216ffb796de06a1a7e9bb947d5a357f3f18ffea41a5611ee3ee0eac"}}, "id": "04c5aad63ba70bc39df10ad208cff66a108e44458e44eea41b305aee7a533877"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.0387409} +{"message": "/usr/lib/tmpfiles.d/abrt.conf:2: Failed to resolve user 'abrt': No such process\n/usr/lib/tmpfiles.d/abrt.conf:9: Failed to resolve user 'abrt': No such process\n", "context": {"origin": "stages/org.osbuild.rpm", "pipeline": {"name": "build", "id": "32e87da44d9a519e89770723a33b7ecdd4ab85b872ae6ab8aaa94bdef9a275c7", "stage": {"name": "org.osbuild.rpm", "id": "bf00d0e1e216ffb796de06a1a7e9bb947d5a357f3f18ffea41a5611ee3ee0eac"}}, "id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.9726782} +{"message": "/usr/lib/tmpfiles.d/gluster.conf:2: Failed to resolve user 'gluster': No such process\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.9730365} +{"message": "/usr/lib/tmpfiles.d/openvpn.conf:1: Failed to resolve group 'openvpn': No such process\n/usr/lib/tmpfiles.d/openvpn.conf:2: Failed to resolve group 'openvpn': No such process\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.973389} +{"message": "/usr/lib/tmpfiles.d/rpcbind.conf:2: Failed to resolve user 'rpc': No such process\n/usr/lib/tmpfiles.d/screen.conf:2: Failed to resolve group 'screen': No such process\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589407.973523} +{"message": "imported gpg key\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589408.1561568} +{"message": "Verifying packages...\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589408.2859807} +{"message": "Preparing packages...\n", "context": {"id": "28cfbf2f2e978d615275d5fdfaab2a4f5a2d85f92accef41421b07d3584fc71a"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 2, "done": 0}}, "timestamp": 1731589408.6820562} +{"message": "yum-4.19.2-1.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.0513315} +{"message": "sssd-kcm-2.9.4-2.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.063613} +{"message": "Created symlink /etc/systemd/system/sockets.target.wants/sssd-kcm.socket \u2192 /usr/lib/systemd/system/sssd-kcm.socket.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.0950909} +{"message": "console-login-helper-messages-motdgen-0.21.3-6.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.096849} +{"message": "grub2-pc-1:2.06-120.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.1272542} +{"message": "rpm-plugin-selinux-4.19.1.1-1.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.140814} +{"message": "shim-x64-15.8-3.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.153876} +{"message": "shim-ia32-15.8-3.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.1752942} +{"message": "audit-3.1.2-8.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.1908624} +{"message": "Created symlink /etc/systemd/system/multi-user.target.wants/auditd.service \u2192 /usr/lib/systemd/system/auditd.service.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.2163906} +{"message": "Running in chroot, ignoring command 'start'\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.2259655} +{"message": "zram-generator-defaults-1.1.2-8.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.2273843} +{"message": "console-login-helper-messages-issuegen-0.21.3-6.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.2394087} +{"message": "passwd-0.80-15.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.2604709} +{"message": "grub2-tools-extra-1:2.06-120.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.27498} +{"message": "console-login-helper-messages-profile-0.21.3-6.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.296905} +{"message": "openssh-clients-9.3p1-10.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.3080597} +{"message": "openssh-server-9.3p1-10.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.5910141} +{"message": "Created symlink /etc/systemd/system/multi-user.target.wants/sshd.service \u2192 /usr/lib/systemd/system/sshd.service.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.6221566} +{"message": "qemu-guest-agent-2:8.1.3-5.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.6235652} +{"message": "Created symlink /etc/systemd/system/dev-virtio\\x2dports-org.qemu.guest_agent.0.device.wants/qemu-guest-agent.service \u2192 /usr/lib/systemd/system/qemu-guest-agent.service.\nUnit /usr/lib/systemd/system/qemu-guest-agent.service is added as a dependency to a non-existent unit dev-virtio\\x2dports-org.qemu.guest_agent.0.device.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.6498303} +{"message": "chrony-4.5-1.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.8656073} +{"message": "Created symlink /etc/systemd/system/multi-user.target.wants/chronyd.service \u2192 /usr/lib/systemd/system/chronyd.service.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.900739} +{"message": "cloud-utils-growpart-0.33-3.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.9026768} +{"message": "systemd-oomd-defaults-254.10-1.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.9159076} +{"message": "dracut-config-generic-059-16.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.930336} +{"message": "rsync-3.3.0-1.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.9440362} +{"message": "man-db-2.11.2-5.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589458.9806342} +{"message": "grub2-tools-efi-1:2.06-120.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.0100884} +{"message": "parted-3.6-2.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.0321343} +{"message": "tar-2:1.35-2.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.0494769} +{"message": "vim-minimal-2:9.1.354-1.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.0658994} +{"message": "langpacks-en-4.0-9.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.085638} +{"message": "efibootmgr-18-4.fc39.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.106354} +{"message": "ncurses-6.4-7.20230520.fc39.1.x86_64\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.1180353} +{"message": "rootfiles-8.1-34.fc39.noarch\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.1340723} +{"message": "'/etc/resolv.conf' -> '../run/systemd/resolve/stub-resolv.conf'\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589459.8034782} +{"message": "grub2-probe: error: failed to get canonical path of `/dev/mapper/home-mvogt'.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589461.4988747} +{"message": "No path or device is specified.\nUsage: grub2-probe [OPTION...] [OPTION]... [PATH|DEVICE]\nTry 'grub2-probe --help' or 'grub2-probe --usage' for more information.\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589461.50024} +{"message": "dracut: No '/dev/log' or 'logger' included for syslog logging\n", "context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589463.2160451} +{"context": {"id": "a8df539116bdb983154447f57d8caedfed1498a464b55e3acb5d2c86c1f7cb39"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 1}}, "timestamp": 1731589489.2805912} +{"message": "Finished module org.osbuild.rpm", "context": {"id": "8cabb87a0d6339a8af1428f0c051707c62979c4b71d7adab02475e2608d63b9e"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 2}}, "timestamp": 1731589489.2968793} +{"message": "Starting module org.osbuild.fix-bls", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.fix-bls", "id": "2dbfea3f718f40d8bb8f47bf41d42ae7cb733fa65a8706eafc56a9a0057afaa3"}}, "id": "bcee346346e9e9f61fc8dad926454ff86067ce99cd8ebb8164c531d2eb4a91fb"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 2}}, "timestamp": 1731589489.2970104} +{"context": {"origin": "stages/org.osbuild.fix-bls", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.fix-bls", "id": "2dbfea3f718f40d8bb8f47bf41d42ae7cb733fa65a8706eafc56a9a0057afaa3"}}, "id": "9e75667964c6f911a33195a09d23d3dba7406c790800de0e6e3a4d37fee5ed86"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 2}}, "timestamp": 1731589489.4580193} +{"message": "Finished module org.osbuild.fix-bls", "context": {"id": "bcee346346e9e9f61fc8dad926454ff86067ce99cd8ebb8164c531d2eb4a91fb"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 3}}, "timestamp": 1731589489.4618332} +{"message": "Starting module org.osbuild.locale", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.locale", "id": "5a02c4b770e29d8a7d8afaf3a99e2af55c6d318626271184b425ea33a94e44d0"}}, "id": "aa24b6e58c0689aa57c2427ca969e23eb34604bf86ac77386caf4b6d4f3dc1e8"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 3}}, "timestamp": 1731589489.4619553} +{"message": "/run/osbuild/tree: /etc/locale.conf written.\n", "context": {"origin": "stages/org.osbuild.locale", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.locale", "id": "5a02c4b770e29d8a7d8afaf3a99e2af55c6d318626271184b425ea33a94e44d0"}}, "id": "54e1b27fe3f2a60d1e0fd1693fc5752679b38912226bf86048b78634eecc7c2c"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 3}}, "timestamp": 1731589489.6036026} +{"context": {"id": "54e1b27fe3f2a60d1e0fd1693fc5752679b38912226bf86048b78634eecc7c2c"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 3}}, "timestamp": 1731589489.626229} +{"message": "Finished module org.osbuild.locale", "context": {"id": "aa24b6e58c0689aa57c2427ca969e23eb34604bf86ac77386caf4b6d4f3dc1e8"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 4}}, "timestamp": 1731589489.629629} +{"message": "Starting module org.osbuild.hostname", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.hostname", "id": "1269a69302b992aab8f82b5c84334187bcf437fcfdcaac2e5e97fb724f08cfb9"}}, "id": "27444c1c8396c38a48dc91e0cad6578c3cefefea01ae0c54aae0a0ad1232a382"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 4}}, "timestamp": 1731589489.6298892} +{"message": "/run/osbuild/tree: /etc/hostname written.\n", "context": {"origin": "stages/org.osbuild.hostname", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.hostname", "id": "1269a69302b992aab8f82b5c84334187bcf437fcfdcaac2e5e97fb724f08cfb9"}}, "id": "ffc3ac0c8a636cf4da6a987a41b82ae74748b118b8fcc01f5a0e8b8447b06416"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 4}}, "timestamp": 1731589489.8085449} +{"context": {"id": "ffc3ac0c8a636cf4da6a987a41b82ae74748b118b8fcc01f5a0e8b8447b06416"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 4}}, "timestamp": 1731589489.8311186} +{"message": "Finished module org.osbuild.hostname", "context": {"id": "27444c1c8396c38a48dc91e0cad6578c3cefefea01ae0c54aae0a0ad1232a382"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 5}}, "timestamp": 1731589489.834136} +{"message": "Starting module org.osbuild.timezone", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.timezone", "id": "8b859c0c66947193b63ccb5b827c804cb875f4e41aeca61f6fcde2d204dc3e8e"}}, "id": "bbabf608db1fc59ec89e5bbf79dcf1eed5964c69cd41344f7b63b1f42de1604d"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 5}}, "timestamp": 1731589489.8342402} +{"message": "/run/osbuild/tree: /etc/localtime written\n", "context": {"origin": "stages/org.osbuild.timezone", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.timezone", "id": "8b859c0c66947193b63ccb5b827c804cb875f4e41aeca61f6fcde2d204dc3e8e"}}, "id": "d6ff784544e543ede80db511fd3378e6732d5b3403c5a9b34bd449d82011949e"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 5}}, "timestamp": 1731589489.9741096} +{"context": {"id": "d6ff784544e543ede80db511fd3378e6732d5b3403c5a9b34bd449d82011949e"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 5}}, "timestamp": 1731589489.995379} +{"message": "Finished module org.osbuild.timezone", "context": {"id": "bbabf608db1fc59ec89e5bbf79dcf1eed5964c69cd41344f7b63b1f42de1604d"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 6}}, "timestamp": 1731589489.9982831} +{"message": "Starting module org.osbuild.fstab", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.fstab", "id": "7e70122b21bd37c954628e236a38a9e4a06ff98304096e79b0fda10efe2666dd"}}, "id": "650e7ec37bc6152436361d93b87ad000ac078c2f37c58863b51495d77827639b"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 6}}, "timestamp": 1731589489.998402} +{"context": {"origin": "stages/org.osbuild.fstab", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.fstab", "id": "7e70122b21bd37c954628e236a38a9e4a06ff98304096e79b0fda10efe2666dd"}}, "id": "4c831725bbb974e02facd575270b3e9934cde2f59520eaa278988b81446e11e5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 6}}, "timestamp": 1731589490.1687038} +{"message": "Finished module org.osbuild.fstab", "context": {"id": "650e7ec37bc6152436361d93b87ad000ac078c2f37c58863b51495d77827639b"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 7}}, "timestamp": 1731589490.1722417} +{"message": "Starting module org.osbuild.grub2", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.grub2", "id": "055599f8b2b051be797a211542627e324b93ba33a3c9b8dcaaa1854181935785"}}, "id": "5719a89889871adbf363583cd0ea77c6d8495b42993cc013307273c767a9dbfc"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 7}}, "timestamp": 1731589490.1723628} +{"message": "# GRUB Environment Block\nsaved_entry=ffffffffffffffffffffffffffffffff-6.8.8-200.fc39.x86_64\n####################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################################\nhybrid boot or unified grub config enabled. Writing alias grub config\n", "context": {"origin": "stages/org.osbuild.grub2", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.grub2", "id": "055599f8b2b051be797a211542627e324b93ba33a3c9b8dcaaa1854181935785"}}, "id": "0af628bcfc9d979e3490a03a63ba7ce6b8dbb4a403b2053616f6981e15de9551"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 7}}, "timestamp": 1731589490.3150368} +{"context": {"id": "0af628bcfc9d979e3490a03a63ba7ce6b8dbb4a403b2053616f6981e15de9551"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 7}}, "timestamp": 1731589490.351013} +{"message": "Finished module org.osbuild.grub2", "context": {"id": "5719a89889871adbf363583cd0ea77c6d8495b42993cc013307273c767a9dbfc"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 8}}, "timestamp": 1731589490.3550842} +{"message": "Starting module org.osbuild.systemd", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.systemd", "id": "377ca48ebe9b7447f2bd4796ee047e89604c5583e4cb3ba4dc344a17e56ab2f7"}}, "id": "282a01784352b5a913976b1a4ea69ca9ff2e172ca04ad4bc6e88360cd757bbe0"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 8}}, "timestamp": 1731589490.35528} +{"message": "Created symlink /run/osbuild/tree/etc/systemd/system/default.target \u2192 /usr/lib/systemd/system/multi-user.target.\n", "context": {"origin": "stages/org.osbuild.systemd", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.systemd", "id": "377ca48ebe9b7447f2bd4796ee047e89604c5583e4cb3ba4dc344a17e56ab2f7"}}, "id": "d736cbfefb3eeb4e61bdaf010febf20372d203c055f485fe578cf2372d1c61f8"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 8}}, "timestamp": 1731589490.5153942} +{"context": {"id": "d736cbfefb3eeb4e61bdaf010febf20372d203c055f485fe578cf2372d1c61f8"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 8}}, "timestamp": 1731589490.5355039} +{"message": "Finished module org.osbuild.systemd", "context": {"id": "282a01784352b5a913976b1a4ea69ca9ff2e172ca04ad4bc6e88360cd757bbe0"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 9}}, "timestamp": 1731589490.5399797} +{"message": "Starting module org.osbuild.test", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.test", "id": "b8a0fbff059367c43f66fa2755a64b722ed52c4761ee401b4c9c148fdff9fc97"}}, "id": "78e3178eb5e72fecbf5a88382d903f2532c07bacc2d69bc152a2486f939a601f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 9}}, "timestamp": 1731589490.5402136} +{"context": {"origin": "stages/org.osbuild.test", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.test", "id": "b8a0fbff059367c43f66fa2755a64b722ed52c4761ee401b4c9c148fdff9fc97"}}, "id": "8f6768ed4efd10f233076510f115dbc11c9b71ffcbfb89123730e1e009c2a11d"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 9}}, "timestamp": 1731589490.7005947} +{"message": "Finished module org.osbuild.test", "context": {"id": "78e3178eb5e72fecbf5a88382d903f2532c07bacc2d69bc152a2486f939a601f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 10}}, "timestamp": 1731589490.704822} +{"message": "Starting module org.osbuild.selinux", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.selinux", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311"}}, "id": "cb7b141b3520e37d9290c2abf26b37a6d1452c0b61b070fb0248ad43b5eb4be6"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 10}}, "timestamp": 1731589490.7050712} +{"context": {"origin": "stages/org.osbuild.selinux", "pipeline": {"name": "os", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311", "stage": {"name": "org.osbuild.selinux", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311"}}, "id": "907732325dbc5ce97aa48f7b0e193c83a8af14ce33403ad520d95f250ec81914"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 10}}, "timestamp": 1731589495.58584} +{"message": "Finished module org.osbuild.selinux", "context": {"id": "cb7b141b3520e37d9290c2abf26b37a6d1452c0b61b070fb0248ad43b5eb4be6"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 2, "progress": {"name": "pipeline: os", "total": 11, "done": 11}}, "timestamp": 1731589495.588855} +{"message": "Finished pipeline os", "context": {"id": "cb7b141b3520e37d9290c2abf26b37a6d1452c0b61b070fb0248ad43b5eb4be6"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3}, "timestamp": 1731589495.5889823} +{"message": "Starting pipeline image", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.selinux", "id": "3b4b32941a016e710b292b4735cde9172cccdc4e6edef442f7808b5a50a10311"}}, "id": "b86979bebac0409b800760a047bdff82a4d0d763365ee0180d7921ed3b122f0f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 0}}, "timestamp": 1731589495.5890737} +{"message": "Starting module org.osbuild.truncate", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.truncate", "id": "f2e1277c7fad9c891bc70a262a954c8f8b39558a6c537cb319366a14bff0df91"}}, "id": "3475c9700c2587373471bffe23b8e92286cc1ee0f549bf25a4424649fd15ad26"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 0}}, "timestamp": 1731589495.5896318} +{"context": {"origin": "stages/org.osbuild.truncate", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.truncate", "id": "f2e1277c7fad9c891bc70a262a954c8f8b39558a6c537cb319366a14bff0df91"}}, "id": "b0b7b1e504e76385dcf56c158ba41b60b0b9dc5ab6ee31968f8e23ec022719e5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 0}}, "timestamp": 1731589495.7504082} +{"message": "Finished module org.osbuild.truncate", "context": {"id": "3475c9700c2587373471bffe23b8e92286cc1ee0f549bf25a4424649fd15ad26"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589495.754751} +{"message": "Starting module org.osbuild.sfdisk", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.sfdisk", "id": "ac9d23a68c231c6586d5dd87ec61da4f899d756f8fd063e06dd48a426b1e43ca"}}, "id": "dfd1e947482aba108aa6e7569910b6a6d968d482514286ad74a04cff83a55d99"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589495.7550597} +{"message": "device/device (org.osbuild.loopback): loop1 acquired (locked: True)\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.sfdisk", "id": "ac9d23a68c231c6586d5dd87ec61da4f899d756f8fd063e06dd48a426b1e43ca"}}, "id": "7226256d5c68706e33b32fb33a6431305684bce5d555e40146f28f9274f6c9a4"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589495.8113465} +{"message": "label: gpt\nlabel-id: D209C89E-EA5E-4FBD-B161-B461CCE297E0\nstart=\"2048\", size=\"2048\", type=\"21686148-6449-6E6F-744E-656564454649\", uuid=\"FAC7F1FB-3E8D-4137-A512-961DE09A5549\", bootable\nstart=\"4096\", size=\"409600\", type=\"C12A7328-F81F-11D2-BA4B-00A0C93EC93B\", uuid=\"68B2905B-DF3E-4FB3-80FA-49D1E773AA33\"\nstart=\"413696\", size=\"1024000\", type=\"0FC63DAF-8483-4772-8E79-3D69D8477DE4\", uuid=\"CB07C243-BC44-4717-853E-28852021225B\"\nstart=\"1437696\", size=\"9048031\", type=\"0FC63DAF-8483-4772-8E79-3D69D8477DE4\", uuid=\"6264D520-3FB9-423F-8AB8-7A0A8E3D3562\"\n", "context": {"origin": "stages/org.osbuild.sfdisk", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.sfdisk", "id": "ac9d23a68c231c6586d5dd87ec61da4f899d756f8fd063e06dd48a426b1e43ca"}}, "id": "927115573dc94c1a06925cf87bd765b94b098edf5fed86c6f84fc518752e9c63"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589495.9657307} +{"message": "{\n \"partitiontable\": {\n \"label\": \"gpt\",\n \"id\": \"D209C89E-EA5E-4FBD-B161-B461CCE297E0\",\n \"device\": \"/dev/loop1\",\n \"unit\": \"sectors\",\n \"firstlba\": 2048,\n \"lastlba\": 10485726,\n \"sectorsize\": 512,\n \"partitions\": [\n {\n \"node\": \"/dev/loop1p1\",\n \"start\": 2048,\n \"size\": 2048,\n \"type\": \"21686148-6449-6E6F-744E-656564454649\",\n \"uuid\": \"FAC7F1FB-3E8D-4137-A512-961DE09A5549\"\n },{\n \"node\": \"/dev/loop1p2\",\n \"start\": 4096,\n \"size\": 409600,\n \"type\": \"C12A7328-F81F-11D2-BA4B-00A0C93EC93B\",\n \"uuid\": \"68B2905B-DF3E-4FB3-80FA-49D1E773AA33\"\n },{\n \"node\": \"/dev/loop1p3\",\n \"start\": 413696,\n \"size\": 1024000,\n \"type\": \"0FC63DAF-8483-4772-8E79-3D69D8477DE4\",\n \"uuid\": \"CB07C243-BC44-4717-853E-28852021225B\"\n },{\n \"node\": \"/dev/loop1p4\",\n \"start\": 1437696,\n \"size\": 9048031,\n \"type\": \"0FC63DAF-8483-4772-8E79-3D69D8477DE4\",\n \"uuid\": \"6264D520-3FB9-423F-8AB8-7A0A8E3D3562\"\n }\n ]\n }\n}\n", "context": {"id": "927115573dc94c1a06925cf87bd765b94b098edf5fed86c6f84fc518752e9c63"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589496.0405648} +{"context": {"id": "927115573dc94c1a06925cf87bd765b94b098edf5fed86c6f84fc518752e9c63"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 1}}, "timestamp": 1731589496.06596} +{"message": "Finished module org.osbuild.sfdisk", "context": {"id": "dfd1e947482aba108aa6e7569910b6a6d968d482514286ad74a04cff83a55d99"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 2}}, "timestamp": 1731589496.084432} +{"message": "Starting module org.osbuild.mkfs.fat", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.fat", "id": "24a19015b6965158279078c42ef7ca183e1d486c3e185b2769b215edb2d9835d"}}, "id": "83adc735a1eef1dbc0a51792fb9823685c3e598145611cd4265e1d626e4093aa"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 2}}, "timestamp": 1731589496.0846913} +{"message": "device/device (org.osbuild.loopback): loop1 acquired (locked: True)\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.fat", "id": "24a19015b6965158279078c42ef7ca183e1d486c3e185b2769b215edb2d9835d"}}, "id": "2a904d04c9f4b5e0f501716a25b6515594d35310c558097313f8fada518eeb74"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 2}}, "timestamp": 1731589496.1364777} +{"message": "mkfs.fat 4.2 (2021-01-31)\n", "context": {"origin": "stages/org.osbuild.mkfs.fat", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.fat", "id": "24a19015b6965158279078c42ef7ca183e1d486c3e185b2769b215edb2d9835d"}}, "id": "33ddcd4d80470cc805d90a9916e705e5ccd78b5a097ddba02f265efe209b91e7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 2}}, "timestamp": 1731589496.29428} +{"context": {"id": "33ddcd4d80470cc805d90a9916e705e5ccd78b5a097ddba02f265efe209b91e7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 2}}, "timestamp": 1731589496.3142166} +{"message": "Finished module org.osbuild.mkfs.fat", "context": {"id": "83adc735a1eef1dbc0a51792fb9823685c3e598145611cd4265e1d626e4093aa"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.3291883} +{"message": "Starting module org.osbuild.mkfs.ext4", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "ecc55150fe888bf60a1c35c2507cfee4c8887b6c17d7cbeb2cadd19123ee3727"}}, "id": "8f71ff666244803f0c279326e886cb4191522e855c5ec8b03ce373d68d15b1e2"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.3293064} +{"message": "device/device (org.osbuild.loopback): loop1 acquired (locked: True)\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "ecc55150fe888bf60a1c35c2507cfee4c8887b6c17d7cbeb2cadd19123ee3727"}}, "id": "85de6b653399d4090ca4e6ba00a2c584ffae40ccf5941fe6a022d4ea1dbec55d"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.3818061} +{"message": "mke2fs 1.47.0 (5-Feb-2023)\n", "context": {"origin": "stages/org.osbuild.mkfs.ext4", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "ecc55150fe888bf60a1c35c2507cfee4c8887b6c17d7cbeb2cadd19123ee3727"}}, "id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5199203} +{"message": "Discarding device blocks: 0/512000\b\b\b\b\b\b\b\b\b\b\b\b\b", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5425177} +{"message": " \b\b\b\b\b\b\b\b\b\b\b\b\bdone \nCreating filesystem with 512000 1k blocks and 128016 inodes\nFilesystem UUID: 0194fdc2-fa2f-4cc0-81d3-ff12045b73c8\nSuperblock backups stored on blocks: \n\t8193, 24577, 40961, 57345, 73729, 204801, 221185, 401409\n\nAllocating group tables: 0/63\b\b\b\b\b ", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5427375} +{"message": "\b\b\b\b\bdone \nWriting inode tables: 0/63\b\b\b\b\b", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5429602} +{"message": " \b\b\b\b\bdone \n", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5431833} +{"message": "Creating journal (8192 blocks): ", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5448349} +{"message": "done\n", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5449948} +{"message": "Writing superblocks and filesystem accounting information: 0/63\b\b\b\b\b", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5458062} +{"message": " \b\b\b\b\b", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5459423} +{"message": "done\n\n", "context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5711417} +{"context": {"id": "9b5093e912668ef3a6fa649c79c350ec5b73ab10dc2b305674d6426bd342d126"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 3}}, "timestamp": 1731589496.5940046} +{"message": "Finished module org.osbuild.mkfs.ext4", "context": {"id": "8f71ff666244803f0c279326e886cb4191522e855c5ec8b03ce373d68d15b1e2"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.6240928} +{"message": "Starting module org.osbuild.mkfs.ext4", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "8faca50cd53f7761453fe8ded47dc70f5fc31289d12115c9670fca495a5ac65a"}}, "id": "cf4709d97e07b3adfce08fedf6dc6c14d31ab86ca405d745278122af9e6b1add"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.6242952} +{"message": "device/device (org.osbuild.loopback): loop1 acquired (locked: True)\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "8faca50cd53f7761453fe8ded47dc70f5fc31289d12115c9670fca495a5ac65a"}}, "id": "aa516f26e1dd759cb39e78e985908b31da856824c9e7b0c4607851c2ead389b6"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.680242} +{"message": "mke2fs 1.47.0 (5-Feb-2023)\n", "context": {"origin": "stages/org.osbuild.mkfs.ext4", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.mkfs.ext4", "id": "8faca50cd53f7761453fe8ded47dc70f5fc31289d12115c9670fca495a5ac65a"}}, "id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.818343} +{"message": "Discarding device blocks: 0/1131003\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b\b\b\b\b\b\b\bdone \nCreating filesystem with 1131003 4k blocks and 282800 inodes\nFilesystem UUID: 6e4ff95f-f662-45ee-a82a-bdf44a2d0b75\nSuperblock backups stored on blocks: \n\t32768, 98304, 163840, 229376, 294912, 819200, 884736\n\nAllocating group tables: 0/35\b\b\b\b\b \b\b\b\b\bdone \n", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.8416996} +{"message": "Writing inode tables: 0/35\b\b\b\b\b \b\b\b\b\bdone \n", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.8418274} +{"message": "Creating journal (16384 blocks): done\n", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.8428683} +{"message": "Writing superblocks and filesystem accounting information: 0/35\b\b\b\b\b", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.843912} +{"message": " \b\b\b\b\b", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.8441703} +{"message": "done\n\n", "context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.8685284} +{"context": {"id": "0422871c9fd81f43a1ee55073e3fc1f476481626b982aba3c5effa59b253bef7"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 4}}, "timestamp": 1731589496.890101} +{"message": "Finished module org.osbuild.mkfs.ext4", "context": {"id": "cf4709d97e07b3adfce08fedf6dc6c14d31ab86ca405d745278122af9e6b1add"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589496.9039688} +{"message": "Starting module org.osbuild.copy", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.copy", "id": "680a8469eb3808b79f4aad84a6e76ab12d1ff6a5ee52f00db67b548b9e308898"}}, "id": "9df0c962cebe1e8d589cc07d7c4f7ae6e5413f258135bb171f5dbbba2aea9e6c"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589496.9041119} +{"message": "device/boot (org.osbuild.loopback): loop1 acquired (locked: False)\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.copy", "id": "680a8469eb3808b79f4aad84a6e76ab12d1ff6a5ee52f00db67b548b9e308898"}}, "id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.02275} +{"message": "device/boot.efi (org.osbuild.loopback): loop2 acquired (locked: False)\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.0713403} +{"message": "device/root (org.osbuild.loopback): loop3 acquired (locked: False)\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.122226} +{"message": "mount/root (org.osbuild.ext4): mounting /dev/loop3 -> /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.1730416} +{"message": "mount/boot (org.osbuild.ext4): mounting /dev/loop1 -> /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/boot\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.2474706} +{"message": "mount/boot.efi (org.osbuild.fat): mounting /dev/loop2 -> /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/boot/efi\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.3091009} +{"message": "copying '/run/osbuild/inputs/root-tree/.' -> '/run/osbuild/mounts/.'\n", "context": {"origin": "stages/org.osbuild.copy", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.copy", "id": "680a8469eb3808b79f4aad84a6e76ab12d1ff6a5ee52f00db67b548b9e308898"}}, "id": "2b74e7fe817f6ba5a1d67b8525560f8c4d3b4eeb944de02590b1fd5852578e53"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589497.461263} +{"context": {"id": "2b74e7fe817f6ba5a1d67b8525560f8c4d3b4eeb944de02590b1fd5852578e53"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589498.3990188} +{"message": "mount/boot.efi (org.osbuild.fat): umount: /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/boot/efi unmounted\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589498.4100862} +{"message": "mount/boot (org.osbuild.ext4): umount: /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/boot unmounted\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589498.5906475} +{"message": "mount/root (org.osbuild.ext4): umount: /home/mvogt/devel/osbuild/osbuild/.osbuild/tmp/buildroot-tmp-kuenr7p2/mounts/ unmounted\n", "context": {"id": "fad4b6354bfb60b380327d3e335d545cc1be0b803435295779f3f13969a2acf5"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 5}}, "timestamp": 1731589499.3330636} +{"message": "Finished module org.osbuild.copy", "context": {"id": "9df0c962cebe1e8d589cc07d7c4f7ae6e5413f258135bb171f5dbbba2aea9e6c"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.4731762} +{"message": "Starting module org.osbuild.grub2.inst", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.grub2.inst", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5"}}, "id": "fa3c23e48f75d29625fcbd6deb24b9523f18ead35888cb6ceced7387a46c697f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.4733772} +{"message": "prefix: (,gpt3)/grub2\n", "context": {"origin": "stages/org.osbuild.grub2.inst", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.grub2.inst", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5"}}, "id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6137075} +{"message": "grub2-mkimage: info: the total module size is 0x5454.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/kernel.img.\n", "context": {"id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6153913} +{"message": "grub2-mkimage: info: locating the section .text at 0x0.\ngrub2-mkimage: info: locating the section .rodata at 0xaf8c.\ngrub2-mkimage: info: locating the section .module_license at 0xcd78.\ngrub2-mkimage: info: locating the section .data at 0xcda0.\ngrub2-mkimage: info: locating the section .bss at 0xd6c0.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/biosdisk.mod.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/part_gpt.mod.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/fshelp.mod.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/ext2.mod.\ngrub2-mkimage: info: kernel_img=0x55f9f4989b60, kernel_size=0xd6b8.\n", "context": {"id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6155815} +{"message": "grub2-mkimage: info: the core size is 0x7299.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/lzma_decompress.img.\ngrub2-mkimage: info: reading /usr/lib/grub/i386-pc/diskboot.img.\ngrub2-mkimage: info: writing 0x200 bytes.\ngrub2-mkimage: info: writing 0x7da9 bytes.\n", "context": {"id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6362607} +{"message": "grub core size is 32681\nwiring core to (2048, 1048576)\nsector start param: 1049076\n", "context": {"id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6472628} +{"context": {"id": "9b48877fa19850adc864504cd4a83049914dd36424534b7b2a7d506de1542afd"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 6}}, "timestamp": 1731589499.6673355} +{"message": "Finished module org.osbuild.grub2.inst", "context": {"id": "fa3c23e48f75d29625fcbd6deb24b9523f18ead35888cb6ceced7387a46c697f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 3, "progress": {"name": "pipeline: image", "total": 7, "done": 7}}, "timestamp": 1731589499.6708198} +{"message": "Finished pipeline image", "context": {"id": "fa3c23e48f75d29625fcbd6deb24b9523f18ead35888cb6ceced7387a46c697f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 4}, "timestamp": 1731589499.671024} +{"message": "manifest ./test/data/manifests/fedora-boot.json finished successfully\n", "context": {"origin": "osbuild.main_cli", "pipeline": {"name": "image", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5", "stage": {"name": "org.osbuild.grub2.inst", "id": "ca2c3347b6a6b5c5c4032dd3175b3462e277262fa5a03dbda5a4fc3d43846fe5"}}, "id": "a56d3a48f6cb5756fe964a29e79aa2e326a5ea66d046a066a0c37356f8c3532f"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 4}, "timestamp": 1731589499.6711605} From c72baff233adb5eb515cdf9660cf48455c55a5a5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 15:56:55 +0100 Subject: [PATCH 02/17] osbuildmonitor: new package to support jsonseq monitoring from osbuild This commit adds support for the jsonseq based monitoring that osbuild provides. This is useful for machine readable status reporting. --- pkg/osbuildmonitor/export_test.go | 5 + pkg/osbuildmonitor/monitor.go | 138 ++++++++++++++++++++++++ pkg/osbuildmonitor/monitor_json_test.go | 55 ++++++++++ pkg/osbuildmonitor/monitor_test.go | 81 ++++++++++++++ 4 files changed, 279 insertions(+) create mode 100644 pkg/osbuildmonitor/export_test.go create mode 100644 pkg/osbuildmonitor/monitor.go create mode 100644 pkg/osbuildmonitor/monitor_json_test.go create mode 100644 pkg/osbuildmonitor/monitor_test.go diff --git a/pkg/osbuildmonitor/export_test.go b/pkg/osbuildmonitor/export_test.go new file mode 100644 index 0000000000..a8d5e52937 --- /dev/null +++ b/pkg/osbuildmonitor/export_test.go @@ -0,0 +1,5 @@ +package osbuildmonitor + +type ( + StatusJSON = statusJSON +) diff --git a/pkg/osbuildmonitor/monitor.go b/pkg/osbuildmonitor/monitor.go new file mode 100644 index 0000000000..10c9071799 --- /dev/null +++ b/pkg/osbuildmonitor/monitor.go @@ -0,0 +1,138 @@ +package osbuildmonitor + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" +) + +// Status is a high level aggregation of the low-level osbuild monitor +// messages. It is more structured and meant to be used by UI frontends. +// +// this is intentionally minimal at the beginning until we figure +// out the best API, exposing the jsonseq direct feels too messy +// and based on what we learn here we may consider tweaking +// the osubild progress + +type Status struct { + // TODO: this will need to include a "log" or "stage-output" + // or something so that the full buildlog can be reconstructed + // from the status streaming + + // Progress contains the current progress. + Progress *Progress +} + +// Progress provides progress information from an osbuild build. +// Each progress can have an arbitrary number of sub-progress information +// +// Note while those can be nested arbitrarly deep in practise +// we are at 2 levels currently: +// 1. overall pipeline progress +// 2. stages inside each pipeline +// +// we might get +// 3. stage progress (e.g. rpm install progress) +// +// in the future +type Progress struct { + // A human readable message about what is doing on + Message string + // The amount of work already done + Done int + // The total amount of work for this (sub)progress + Total int + + SubProgress *Progress +} + +// NewStatusScanner returns a StatusScanner that can parse osbuild +// jsonseq monitor status messages +func NewStatusScanner(r io.Reader) *StatusScanner { + return &StatusScanner{ + scanner: bufio.NewScanner(r), + pipelineContextMap: make(map[string]*contextJSON), + } +} + +// StatusScanner scan scan the osbuild jsonseq monitor output +type StatusScanner struct { + scanner *bufio.Scanner + pipelineContextMap map[string]*contextJSON +} + +// Status returns a single status struct from the scanner or nil +// if the end of the status reporting is reached. +func (sr *StatusScanner) Status() (*Status, error) { + if !sr.scanner.Scan() { + return nil, sr.scanner.Err() + } + + var status statusJSON + line := sr.scanner.Bytes() + line = bytes.Trim(line, "\x1e") + if err := json.Unmarshal(line, &status); err != nil { + return nil, fmt.Errorf("cannto scan line %q: %w", line, err) + } + // keep track of the context + // XXX: needs a test + id := status.Context.Pipeline.ID + pipelineContext := sr.pipelineContextMap[id] + if pipelineContext == nil { + sr.pipelineContextMap[id] = &status.Context + } + + st := &Status{ + Progress: &Progress{ + Done: status.Progress.Done, + Total: status.Progress.Total, + }, + } + // add subprogress + prog := st.Progress + for subProg := status.Progress.SubProgress; subProg != nil; subProg = subProg.SubProgress { + prog.SubProgress = &Progress{ + Done: subProg.Done, + Total: subProg.Total, + } + prog = prog.SubProgress + } + + return st, nil +} + +// statusJSON is a single status entry from the osbuild monitor +type statusJSON struct { + Context contextJSON `json:"context"` + Progress progressJSON `json:"progress"` + // Add "Result" here once + // https://github.com/osbuild/osbuild/pull/1831 is merged + + Message string `json:"message"` +} + +// contextJSON is the context for which a status is given. Once a context +// was sent to the user from then on it is only referenced by the ID +type contextJSON struct { + Origin string `json:"origin"` + Pipeline struct { + ID string `json:"id"` + Name string `json:"name"` + Stage struct { + Name string `json:"name"` + ID string `json:"id"` + } `json:"stage"` + } `json:"pipeline"` +} + +// progress is the progress information associcated with a given status. +// The details about nesting are the same as for "Progress" above. +type progressJSON struct { + Name string `json:"name"` + Total int `json:"total"` + Done int `json:"done"` + + SubProgress *progressJSON `json:"progress"` +} diff --git a/pkg/osbuildmonitor/monitor_json_test.go b/pkg/osbuildmonitor/monitor_json_test.go new file mode 100644 index 0000000000..a7ae069b8d --- /dev/null +++ b/pkg/osbuildmonitor/monitor_json_test.go @@ -0,0 +1,55 @@ +package osbuildmonitor_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/osbuildmonitor" +) + +const osbuildMonitorJSON_1 = ` +{ + "message": "Top level message", + "context": { + "origin": "osbuild.monitor", + "pipeline": { + "name": "source org.osbuild.curl", + "id": "598849389c35f93efe2412446f5ca6919434417b9bcea040ea5f9203de81db2c", + "stage": {} + }, + "id": "69816755441434713b7567970edfdd42d58193f163e1fdd506274d52246e87f2" + }, + "timestamp": 1731585664.9090264, + "progress": { + "name": "name", + "total": 4, + "done": 1, + "progress": { + "name": "nested-name", + "total": 8, + "done": 2, + "progress": { + "name": "nested-nested-name", + "total": 16, + "done": 4 + } + } + } +}` + +func TestOsbuildStatusNestingWorks(t *testing.T) { + var status osbuildmonitor.StatusJSON + + err := json.Unmarshal([]byte(osbuildMonitorJSON_1), &status) + assert.NoError(t, err) + assert.Equal(t, "Top level message", status.Message) + assert.Equal(t, "name", status.Progress.Name) + assert.Equal(t, 4, status.Progress.Total) + assert.Equal(t, "nested-name", status.Progress.SubProgress.Name) + assert.Equal(t, 8, status.Progress.SubProgress.Total) + assert.Equal(t, "nested-nested-name", status.Progress.SubProgress.SubProgress.Name) + assert.Equal(t, 16, status.Progress.SubProgress.SubProgress.Total) + assert.Nil(t, status.Progress.SubProgress.SubProgress.SubProgress) +} diff --git a/pkg/osbuildmonitor/monitor_test.go b/pkg/osbuildmonitor/monitor_test.go new file mode 100644 index 0000000000..edbe342188 --- /dev/null +++ b/pkg/osbuildmonitor/monitor_test.go @@ -0,0 +1,81 @@ +package osbuildmonitor_test + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/images/pkg/osbuildmonitor" +) + +const osbuildMonitorLines_curl = `{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/k/kpartx-0.9.5-2.fc39.x86_64.rpm\n", "context": {"origin": "org.osbuild", "pipeline": {"name": "source org.osbuild.curl", "id": "598849389c35f93efe2412446f5ca6919434417b9bcea040ea5f9203de81db2c", "stage": {}}, "id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8252223} +{"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/l/langpacks-fonts-en-4.0-9.fc39.noarch.rpm\n", "context": {"id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8256931}` + +func TestScannerSimple(t *testing.T) { + r := bytes.NewBufferString(osbuildMonitorLines_curl) + scanner := osbuildmonitor.NewStatusScanner(r) + // first line + st, err := scanner.Status() + assert.NoError(t, err) + assert.Equal(t, &osbuildmonitor.Status{ + Progress: &osbuildmonitor.Progress{ + Done: 0, + Total: 4, + }, + }, st) + // second line + st, err = scanner.Status() + assert.NoError(t, err) + assert.Equal(t, &osbuildmonitor.Status{ + Progress: &osbuildmonitor.Progress{ + Done: 0, + Total: 4, + }, + }, st) + // end + st, err = scanner.Status() + assert.NoError(t, err) + assert.Nil(t, st) +} + +const osbuildMontiorLines_subprogress = `{"message": "Starting module org.osbuild.rpm", "context": {"origin": "osbuild.monitor", "pipeline": {"name": "build", "id": "32e87da44d9a519e89770723a33b7ecdd4ab85b872ae6ab8aaa94bdef9a275c7", "stage": {"name": "org.osbuild.rpm", "id": "bf00d0e1e216ffb796de06a1a7e9bb947d5a357f3f18ffea41a5611ee3ee0eac"}}, "id": "04c5aad63ba70bc39df10ad208cff66a108e44458e44eea41b305aee7a533877"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 1, "progress": {"name": "pipeline: build", "total": 8, "done": 2, "progress": {"name": "sub-sub-progress", "total": 16, "done": 4}}}, "timestamp": 1731600115.148399} +` + +func TestScannerSubprogress(t *testing.T) { + r := bytes.NewBufferString(osbuildMontiorLines_subprogress) + scanner := osbuildmonitor.NewStatusScanner(r) + st, err := scanner.Status() + assert.NoError(t, err) + assert.Equal(t, &osbuildmonitor.Status{ + Progress: &osbuildmonitor.Progress{ + Done: 1, + Total: 4, + SubProgress: &osbuildmonitor.Progress{ + Done: 2, + Total: 8, + SubProgress: &osbuildmonitor.Progress{ + Done: 4, + Total: 16, + }, + }, + }, + }, st) +} + +func TestScannerSmoke(t *testing.T) { + f, err := os.Open("testdata/osbuild-monitor-output.json") + require.NoError(t, err) + defer f.Close() + + scanner := osbuildmonitor.NewStatusScanner(f) + for { + st, err := scanner.Status() + assert.NoError(t, err) + if st == nil { + break + } + } +} From c8cb119aded0e07961dd5c720f3558f0a4e3490f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 15:16:45 +0100 Subject: [PATCH 03/17] osbuildmonitor: include a "Trace" string in Status This commit adds a `Status.Trace` string in the osbuildmonitor. Trace contains a single log line, usually very low-level or stage output bug useful for e.g. bug reporting. Should in general not be displayed to the user but the concatenation of all "trace" lines should give the same information as running osbuild on a terminal. --- pkg/osbuildmonitor/monitor.go | 12 +++++++++--- pkg/osbuildmonitor/monitor_test.go | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/osbuildmonitor/monitor.go b/pkg/osbuildmonitor/monitor.go index 10c9071799..2ea7ef06c7 100644 --- a/pkg/osbuildmonitor/monitor.go +++ b/pkg/osbuildmonitor/monitor.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "strings" ) // Status is a high level aggregation of the low-level osbuild monitor @@ -17,9 +18,12 @@ import ( // the osubild progress type Status struct { - // TODO: this will need to include a "log" or "stage-output" - // or something so that the full buildlog can be reconstructed - // from the status streaming + // Trace contains a single log line, usually very low-level or + // stage output bug useful for e.g. bug reporting. Should in + // general not be displayed to the user but the concatenation + // of all "trace" lines should give the same information as + // running osbuild on a terminal + Trace string // Progress contains the current progress. Progress *Progress @@ -85,11 +89,13 @@ func (sr *StatusScanner) Status() (*Status, error) { } st := &Status{ + Trace: strings.TrimSpace(status.Message), Progress: &Progress{ Done: status.Progress.Done, Total: status.Progress.Total, }, } + // add subprogress prog := st.Progress for subProg := status.Progress.SubProgress; subProg != nil; subProg = subProg.SubProgress { diff --git a/pkg/osbuildmonitor/monitor_test.go b/pkg/osbuildmonitor/monitor_test.go index edbe342188..d856767ded 100644 --- a/pkg/osbuildmonitor/monitor_test.go +++ b/pkg/osbuildmonitor/monitor_test.go @@ -21,6 +21,7 @@ func TestScannerSimple(t *testing.T) { st, err := scanner.Status() assert.NoError(t, err) assert.Equal(t, &osbuildmonitor.Status{ + Trace: "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/k/kpartx-0.9.5-2.fc39.x86_64.rpm", Progress: &osbuildmonitor.Progress{ Done: 0, Total: 4, @@ -30,6 +31,7 @@ func TestScannerSimple(t *testing.T) { st, err = scanner.Status() assert.NoError(t, err) assert.Equal(t, &osbuildmonitor.Status{ + Trace: "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/l/langpacks-fonts-en-4.0-9.fc39.noarch.rpm", Progress: &osbuildmonitor.Progress{ Done: 0, Total: 4, @@ -50,6 +52,7 @@ func TestScannerSubprogress(t *testing.T) { st, err := scanner.Status() assert.NoError(t, err) assert.Equal(t, &osbuildmonitor.Status{ + Trace: "Starting module org.osbuild.rpm", Progress: &osbuildmonitor.Progress{ Done: 1, Total: 4, From 1da14469f1bc9b0fe84663be0972f4b4d1e57599 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 15:46:12 +0100 Subject: [PATCH 04/17] osbuildmonitor: include `Timestamp` for better tracability This commit adds `Status.Timestamp` so that it's easier to correlate what took how long when reconstructing the log (e.g. for bugreporting). See also: https://github.com/osbuild/images/compare/main...mvo5:osbuildmonitor-testdata?expand=1 for the full log. --- pkg/osbuildmonitor/monitor.go | 9 ++++++++- pkg/osbuildmonitor/monitor_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/osbuildmonitor/monitor.go b/pkg/osbuildmonitor/monitor.go index 2ea7ef06c7..adbbf48d61 100644 --- a/pkg/osbuildmonitor/monitor.go +++ b/pkg/osbuildmonitor/monitor.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "strings" + "time" ) // Status is a high level aggregation of the low-level osbuild monitor @@ -27,6 +28,9 @@ type Status struct { // Progress contains the current progress. Progress *Progress + + // Timestamp contains the timestamp the message was recieved in + Timestamp time.Time } // Progress provides progress information from an osbuild build. @@ -87,6 +91,7 @@ func (sr *StatusScanner) Status() (*Status, error) { if pipelineContext == nil { sr.pipelineContextMap[id] = &status.Context } + ts := time.UnixMilli(int64(status.Timestamp * 1000)) st := &Status{ Trace: strings.TrimSpace(status.Message), @@ -94,6 +99,7 @@ func (sr *StatusScanner) Status() (*Status, error) { Done: status.Progress.Done, Total: status.Progress.Total, }, + Timestamp: ts, } // add subprogress @@ -116,7 +122,8 @@ type statusJSON struct { // Add "Result" here once // https://github.com/osbuild/osbuild/pull/1831 is merged - Message string `json:"message"` + Message string `json:"message"` + Timestamp float64 `json:"timestamp"` } // contextJSON is the context for which a status is given. Once a context diff --git a/pkg/osbuildmonitor/monitor_test.go b/pkg/osbuildmonitor/monitor_test.go index d856767ded..a545a7490a 100644 --- a/pkg/osbuildmonitor/monitor_test.go +++ b/pkg/osbuildmonitor/monitor_test.go @@ -4,6 +4,7 @@ import ( "bytes" "os" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,6 +16,9 @@ const osbuildMonitorLines_curl = `{"message": "source/org.osbuild.curl (org.osb {"message": "source/org.osbuild.curl (org.osbuild.curl): Downloaded https://rpmrepo.osbuild.org/v2/mirror/public/f39/f39-x86_64-fedora-20231109/Packages/l/langpacks-fonts-en-4.0-9.fc39.noarch.rpm\n", "context": {"id": "7355d3857aa5c7b3a0c476c13d4b242a625fe190f2e7796df2335f3a34429db3"}, "progress": {"name": "pipelines/sources", "total": 4, "done": 0}, "timestamp": 1731589338.8256931}` func TestScannerSimple(t *testing.T) { + ts1 := 1731589338.8252223 * 1000 + ts2 := 1731589338.8256931 * 1000 + r := bytes.NewBufferString(osbuildMonitorLines_curl) scanner := osbuildmonitor.NewStatusScanner(r) // first line @@ -26,6 +30,7 @@ func TestScannerSimple(t *testing.T) { Done: 0, Total: 4, }, + Timestamp: time.UnixMilli(int64(ts1)), }, st) // second line st, err = scanner.Status() @@ -36,6 +41,7 @@ func TestScannerSimple(t *testing.T) { Done: 0, Total: 4, }, + Timestamp: time.UnixMilli(int64(ts2)), }, st) // end st, err = scanner.Status() @@ -47,6 +53,8 @@ const osbuildMontiorLines_subprogress = `{"message": "Starting module org.osbui ` func TestScannerSubprogress(t *testing.T) { + ts1 := 1731600115.14839 * 1000 + r := bytes.NewBufferString(osbuildMontiorLines_subprogress) scanner := osbuildmonitor.NewStatusScanner(r) st, err := scanner.Status() @@ -65,6 +73,7 @@ func TestScannerSubprogress(t *testing.T) { }, }, }, + Timestamp: time.UnixMilli(int64(ts1)), }, st) } @@ -80,5 +89,6 @@ func TestScannerSmoke(t *testing.T) { if st == nil { break } + assert.NotEqual(t, time.Time{}, st.Timestamp) } } From 067d393ce6f24075110ea7d47d56cd1da9717ead Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 30 Oct 2024 13:26:48 +0100 Subject: [PATCH 05/17] image-builder: add new filters capbility to filter on distro This commit adds support cabilities to filter the list of distros based on a given search expression. Those are: ``` foo name:foo arch:i386 type:ami bootmode:uefi ``` and they will filter the list of distros. --- cmd/image-builder/filters.go | 130 ++++++++++++++++++++++++++++++ cmd/image-builder/filters_test.go | 62 ++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 cmd/image-builder/filters.go create mode 100644 cmd/image-builder/filters_test.go diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go new file mode 100644 index 0000000000..6233e69d2a --- /dev/null +++ b/cmd/image-builder/filters.go @@ -0,0 +1,130 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" + + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distrofactory" +) + +// XXX: move this file into distro factory? + +type FilterResult struct { + Distro distro.Distro + Arch distro.Arch + ImgType distro.ImageType +} + +func FilterDistros(fac *distrofactory.Factory, distroNames []string, filters Filters) ([]FilterResult, error) { + var res []FilterResult + + if err := distro.SortNames(distroNames); err != nil { + return nil, err + } + for _, distroName := range distroNames { + distro := fac.GetDistro(distroName) + if distro == nil { + logrus.Debugf("skipping %v: has repositories but unsupported", distroName) + } + for _, archName := range distro.ListArches() { + a, err := distro.GetArch(archName) + if err != nil { + return nil, err + } + for _, imgTypeName := range a.ListImageTypes() { + imgType, err := a.GetImageType(imgTypeName) + if err != nil { + return nil, err + } + if filters.Matches(distro, a, imgType) { + res = append(res, FilterResult{distro, a, imgType}) + } + } + } + } + + return res, nil +} + +// Filters is a way to filter a list of distros +type Filters []filter + +// NewFilters creates a filtering for a list of distros +func NewFilters(sl []string) (Filters, error) { + var filters []filter + for _, s := range sl { + l := strings.SplitN(s, ":", 2) + switch l[0] { + case s: + filters = append(filters, &distroNameFilter{ + filter: l[0], + exact: false, + }) + case "name": + filters = append(filters, &distroNameFilter{l[1], true}) + case "arch": + filters = append(filters, &archFilter{l[1]}) + case "type": + filters = append(filters, &imgTypeFilter{l[1]}) + // mostly here to show how powerful this is + case "bootmode": + filters = append(filters, &bootmodeFilter{l[1]}) + default: + return nil, fmt.Errorf("unsupported filter prefix: %q", l[0]) + } + } + return filters, nil +} + +// Matches returns true if the given (distro,arch,imgType) tuple matches +// the filter expressions +func (fl Filters) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { + matches := true + for _, f := range fl { + matches = matches && f.Matches(distro, arch, imgType) + } + return matches +} + +type filter interface { + Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool +} + +type distroNameFilter struct { + filter string + exact bool +} + +func (d *distroNameFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { + if d.exact { + return distro.Name() == d.filter + } + return strings.Contains(distro.Name(), d.filter) +} + +type archFilter struct { + filter string +} + +func (d *archFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { + return strings.Contains(arch.Name(), d.filter) +} + +type imgTypeFilter struct { + filter string +} + +func (d *imgTypeFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { + return strings.Contains(imgType.Name(), d.filter) +} + +type bootmodeFilter struct { + filter string +} + +func (d *bootmodeFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { + return imgType.BootMode().String() == d.filter +} diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go new file mode 100644 index 0000000000..6cb49e7665 --- /dev/null +++ b/cmd/image-builder/filters_test.go @@ -0,0 +1,62 @@ +package main_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/images/cmd/image-builder" + "github.com/osbuild/images/pkg/distrofactory" +) + +func TestDistroFilter(t *testing.T) { + fac := distrofactory.NewTestDefault() + + for _, tc := range []struct { + searchExpr []string + distro, arch, imgType string + expectsMatch bool + }{ + // simple (fuzzy on name) + {[]string{"foo"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"test-distro-1"}, "test-distro-1", "test_arch3", "qcow2", true}, + {[]string{"test-distro"}, "test-distro-1", "test_arch3", "qcow2", true}, + // name: prefix (exact matches only) + {[]string{"name:bar"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"name:test-distro-1"}, "test-distro-1", "test_arch3", "qcow2", true}, + {[]string{"name:test-distro"}, "test-distro-1", "test_arch3", "qcow2", false}, + // arch: prefix + // XXX: should we be exact there? what do users expect? + {[]string{"arch:amd64"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"arch:test_arch"}, "test-distro-1", "test_arch3", "qcow2", true}, + {[]string{"arch:test_ar"}, "test-distro-1", "test_arch3", "qcow2", true}, + // type: prefix + // XXX: should we be exact there? what do users expect? + {[]string{"type:ami"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", true}, + {[]string{"type:qcow"}, "test-distro-1", "test_arch3", "qcow2", true}, + // bootmode: prefix + {[]string{"bootmode:uefi"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"bootmode:hybrid"}, "test-distro-1", "test_arch3", "qcow2", true}, + // multiple filters are AND + {[]string{"name:test-distro-1", "type:ami"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"name:test-distro-1", "type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", true}, + {[]string{"name:test-distro-1", "arch:amd64", "type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", false}, + } { + // XXX: it would be nice if TestDistro would support constructing + // like GetDistro("rhel-8.1:i386,amd64:ami,qcow2") instead of + // the current very static setup + di := fac.GetDistro(tc.distro) + require.NotNil(t, di) + ar, err := di.GetArch(tc.arch) + require.NoError(t, err) + im, err := ar.GetImageType(tc.imgType) + require.NoError(t, err) + ff, err := main.NewFilters(tc.searchExpr) + require.NoError(t, err) + + match := ff.Matches(di, ar, im) + assert.Equal(t, tc.expectsMatch, match, tc) + } +} From acfb94fbd6132341a0a150e7c2735aa5233a4c59 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 28 Oct 2024 10:20:53 +0100 Subject: [PATCH 06/17] cmd: add new image-builder binary and initial `list-images` cmd This commit adds a new `image-builder` binary that will (eventually) be capable to build images standalone (daemonless). There are various use-cases for this, the most important one is CI/CD pipelines but also providing a common abstraction over the "images" library (and potentially later "otk"). This commit implements basic a `list-images` command with filtering and text and json output, e.g. ``` $ image-builder list-images --filter bootmode:none --filter name:fedora-39 --filter arch:s390x fedora-39 --arch s390x --type container $ ./image-builder list-images --filter bootmode:none --filter name:fedora-39 --filter arch:s390x --format=json {"distro":{"name":"fedora-39"},"arch":{"name":"s390x"},"image_type":{"name":"container"}} ``` --- cmd/image-builder/export_test.go | 26 ++++++++++++++ cmd/image-builder/filters.go | 3 +- cmd/image-builder/list_images.go | 48 +++++++++++++++++++++++++ cmd/image-builder/main.go | 61 ++++++++++++++++++++++++++++++++ cmd/image-builder/main_test.go | 26 ++++++++++++++ 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 cmd/image-builder/export_test.go create mode 100644 cmd/image-builder/list_images.go create mode 100644 cmd/image-builder/main.go create mode 100644 cmd/image-builder/main_test.go diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go new file mode 100644 index 0000000000..11134c3fe4 --- /dev/null +++ b/cmd/image-builder/export_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "io" + "os" +) + +func MockOsArgs(new []string) (restore func()) { + saved := os.Args + os.Args = append([]string{"argv0"}, new...) + return func() { + os.Args = saved + } +} + +func MockOsStdout(new io.Writer) (restore func()) { + saved := osStdout + osStdout = new + return func() { + osStdout = saved + } +} + +var ( + Run = run +) diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 6233e69d2a..55c98549ee 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -8,6 +8,7 @@ import ( "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/distrofactory" + "github.com/osbuild/images/pkg/distrosort" ) // XXX: move this file into distro factory? @@ -21,7 +22,7 @@ type FilterResult struct { func FilterDistros(fac *distrofactory.Factory, distroNames []string, filters Filters) ([]FilterResult, error) { var res []FilterResult - if err := distro.SortNames(distroNames); err != nil { + if err := distrosort.Names(distroNames); err != nil { return nil, err } for _, distroName := range distroNames { diff --git a/cmd/image-builder/list_images.go b/cmd/image-builder/list_images.go new file mode 100644 index 0000000000..edba8c2554 --- /dev/null +++ b/cmd/image-builder/list_images.go @@ -0,0 +1,48 @@ +package main + +import ( + "io" + "os" + "strings" + + "github.com/osbuild/images/pkg/distrofactory" + "github.com/osbuild/images/pkg/reporegistry" +) + +// XXX: copied from "composer", should be exported there so +// that we keep this in sync +// XXX2: means we need to depend on osbuild-composer-common or something +var repositoryConfigs = []string{ + "/etc/osbuild-composer", + "/usr/share/osbuild-composer", +} + +func listImages(out io.Writer, format string, filterExprs []string) error { + // useful for development/debugging, e.g. run: + // go build && IMAGE_BUILDER_EXTRA_REPOS_PATH=../../test/data ./image-builder + if extraReposPath := os.Getenv("IMAGE_BUILDER_EXTRA_REPOS_PATH"); extraReposPath != "" { + repositoryConfigs = append(repositoryConfigs, strings.Split(extraReposPath, ":")...) + } + + repos, err := reporegistry.New(repositoryConfigs) + if err != nil { + return err + } + filters, err := NewFilters(filterExprs) + if err != nil { + return err + } + fac := distrofactory.NewDefault() + filteredResult, err := FilterDistros(fac, repos.ListDistros(), filters) + if err != nil { + return err + } + + fmter, err := NewFilteredResultFormatter(format) + if err != nil { + return err + } + fmter.Output(out, filteredResult) + + return nil +} diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go new file mode 100644 index 0000000000..f8ffa57537 --- /dev/null +++ b/cmd/image-builder/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "io" + "log" + "os" + + "github.com/sirupsen/logrus" + + "github.com/spf13/cobra" +) + +var osStdout io.Writer = os.Stdout + +func cmdListImages(cmd *cobra.Command, args []string) error { + filter, err := cmd.Flags().GetStringArray("filter") + if err != nil { + return err + } + format, err := cmd.Flags().GetString("format") + if err != nil { + return err + } + + return listImages(osStdout, format, filter) +} + +func run() error { + // images logs a bunch of stuff to Debug/Info that we we do not + // want to show + logrus.SetLevel(logrus.WarnLevel) + + rootCmd := &cobra.Command{ + Use: "image-builder", + Short: "Build operating system images from a given blueprint", + Long: `Build operating system images from a given blueprint + +Image-builder builds operating system images for a range of predefined +operating sytsems like centos and RHEL with easy customizations support.`, + } + + // XXX: this will list 802 images right now, we need a sensible + // default here, maybe without --filter just list all available + // distro names? + listImagesCmd := &cobra.Command{ + Use: "list-images", + RunE: cmdListImages, + SilenceUsage: true, + } + listImagesCmd.Flags().StringArray("filter", nil, "Filter distributions by a specific criteria") + listImagesCmd.Flags().String("format", "", "Output in a specific format (text,json)") + rootCmd.AddCommand(listImagesCmd) + + return rootCmd.Execute() +} + +func main() { + if err := run(); err != nil { + log.Fatalf("error: %s", err) + } +} diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go new file mode 100644 index 0000000000..3571cc8ff4 --- /dev/null +++ b/cmd/image-builder/main_test.go @@ -0,0 +1,26 @@ +package main_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/cmd/image-builder" +) + +func TestListImagesSmoke(t *testing.T) { + t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data") + + restore := main.MockOsArgs([]string{"list-images"}) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err := main.Run() + assert.NoError(t, err) + // output is sorted + assert.Regexp(t, `(?ms)rhel-8.9.*rhel-8.10`, fakeStdout.String()) +} From 9d16119def2d6a7be95ced6d5cf9e80c5a31726e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 30 Oct 2024 18:38:02 +0100 Subject: [PATCH 07/17] cli: implement messy/untested "image-builder manifest" --- cmd/image-builder/list_images.go | 19 +++++-- cmd/image-builder/main.go | 27 ++++++++- cmd/image-builder/manifest.go | 94 ++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 cmd/image-builder/manifest.go diff --git a/cmd/image-builder/list_images.go b/cmd/image-builder/list_images.go index edba8c2554..18892747ea 100644 --- a/cmd/image-builder/list_images.go +++ b/cmd/image-builder/list_images.go @@ -17,23 +17,32 @@ var repositoryConfigs = []string{ "/usr/share/osbuild-composer", } -func listImages(out io.Writer, format string, filterExprs []string) error { +// XXX: move to pkg/reporegistry +func newRepoRegistry() (*reporegistry.RepoRegistry, error) { // useful for development/debugging, e.g. run: // go build && IMAGE_BUILDER_EXTRA_REPOS_PATH=../../test/data ./image-builder if extraReposPath := os.Getenv("IMAGE_BUILDER_EXTRA_REPOS_PATH"); extraReposPath != "" { repositoryConfigs = append(repositoryConfigs, strings.Split(extraReposPath, ":")...) } - repos, err := reporegistry.New(repositoryConfigs) + return reporegistry.New(repositoryConfigs) +} + +func getFilteredImages(filterExprs []string) ([]FilterResult, error) { + repos, err := newRepoRegistry() if err != nil { - return err + return nil, err } filters, err := NewFilters(filterExprs) if err != nil { - return err + return nil, err } fac := distrofactory.NewDefault() - filteredResult, err := FilterDistros(fac, repos.ListDistros(), filters) + return FilterDistros(fac, repos.ListDistros(), filters) +} + +func listImages(out io.Writer, format string, filterExprs []string) error { + filteredResult, err := getFilteredImages(filterExprs) if err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index f8ffa57537..dfed369cdf 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -6,8 +6,9 @@ import ( "os" "github.com/sirupsen/logrus" - "github.com/spf13/cobra" + + "github.com/osbuild/images/pkg/arch" ) var osStdout io.Writer = os.Stdout @@ -25,6 +26,19 @@ func cmdListImages(cmd *cobra.Command, args []string) error { return listImages(osStdout, format, filter) } +func cmdManifest(cmd *cobra.Command, args []string) error { + distroName := args[0] + imgType := args[1] + var archStr string + if len(args) > 2 { + archStr = args[2] + } else { + archStr = arch.Current().String() + } + + return outputManifest(osStdout, distroName, imgType, archStr) +} + func run() error { // images logs a bunch of stuff to Debug/Info that we we do not // want to show @@ -44,6 +58,7 @@ operating sytsems like centos and RHEL with easy customizations support.`, // distro names? listImagesCmd := &cobra.Command{ Use: "list-images", + Short: "List buildable images, use --filter to limit further", RunE: cmdListImages, SilenceUsage: true, } @@ -51,6 +66,16 @@ operating sytsems like centos and RHEL with easy customizations support.`, listImagesCmd.Flags().String("format", "", "Output in a specific format (text,json)") rootCmd.AddCommand(listImagesCmd) + manifestCmd := &cobra.Command{ + Use: "manifest []", + Short: "Build manifest for the given distro/image-type, e.g. centos-9 qcow2", + RunE: cmdManifest, + SilenceUsage: true, + Args: cobra.MinimumNArgs(2), + Hidden: true, + } + rootCmd.AddCommand(manifestCmd) + return rootCmd.Execute() } diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go new file mode 100644 index 0000000000..5f2c22308a --- /dev/null +++ b/cmd/image-builder/manifest.go @@ -0,0 +1,94 @@ +package main + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" +) + +// XXX: duplicated from cmd/build/main.go:depsolve (and probably more places) +func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range packageSets { + res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) + if err != nil { + return nil, nil, err + } + depsolvedSets[name] = res.Packages + repoSets[name] = res.Repos + } + return depsolvedSets, repoSets, nil +} + +func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error { + filterExprs := []string{ + fmt.Sprintf("name:%s", distroName), + fmt.Sprintf("arch:%s", archStr), + fmt.Sprintf("type:%s", imgTypeStr), + } + filteredResults, err := getFilteredImages(filterExprs) + if err != nil { + return err + } + switch len(filteredResults) { + case 0: + return fmt.Errorf("cannot find image for %s %s %s", distroName, imgTypeStr, archStr) + case 1: + // nothing + default: + return fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr) + } + + var bp blueprint.Blueprint + // XXX: what/how much do we expose here? + options := distro.ImageOptions{} + distro := filteredResults[0].Distro + imgType := filteredResults[0].ImgType + + reporeg, err := newRepoRegistry() + if err != nil { + return err + } + repos, err := reporeg.ReposByImageTypeName(distroName, archStr, imgTypeStr) + if err != nil { + return err + } + preManifest, warnings, err := imgType.Manifest(&bp, options, repos, 0) + if err != nil { + return err + } + if len(warnings) > 0 { + // XXX: what can we do here? for things like json output? + // what are these warnings? + return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) + } + // XXX: cleanup, use shared dir,etc + cacheDir, err := os.MkdirTemp("", "depsolve") + if err != nil { + return err + } + packageSpecs, _, err := depsolve(cacheDir, preManifest.GetPackageSetChains(), distro, archStr) + if err != nil { + return err + } + if packageSpecs == nil { + return fmt.Errorf("depsolve did not return any packages") + } + // XXX: support commit/container specs + mf, err := preManifest.Serialize(packageSpecs, nil, nil, nil) + if err != nil { + return err + } + fmt.Fprintf(out, "%s\n", mf) + + return nil +} From 2d451c78b4d9df3179535d4875236ce59db69d87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 30 Oct 2024 18:56:54 +0100 Subject: [PATCH 08/17] cli: implement messy/untested "image-builder build" --- cmd/image-builder/build.go | 36 +++++++++++++++++++++++++++++++++++ cmd/image-builder/filters.go | 21 ++++++++++++++++++++ cmd/image-builder/main.go | 17 +++++++++++++++++ cmd/image-builder/manifest.go | 28 ++++++++------------------- 4 files changed, 82 insertions(+), 20 deletions(-) create mode 100644 cmd/image-builder/build.go diff --git a/cmd/image-builder/build.go b/cmd/image-builder/build.go new file mode 100644 index 0000000000..ea1b081502 --- /dev/null +++ b/cmd/image-builder/build.go @@ -0,0 +1,36 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/osbuild" +) + +func buildImage(out io.Writer, distroName, imgTypeStr string) error { + // cross arch building is not possible, we would have to download + // a pre-populated buildroot (tar,container) with rpm for that + archStr := arch.Current().String() + filterResult, err := getOneImage(distroName, imgTypeStr, archStr) + if err != nil { + return err + } + imgType := filterResult.ImgType + + var mf bytes.Buffer + if err := outputManifest(&mf, distroName, imgTypeStr, archStr); err != nil { + return err + } + + osbuildStoreDir := ".store" + outputDir := "." + buildName := fmt.Sprintf("%s-%s-%s", distroName, imgTypeStr, archStr) + jobOutputDir := filepath.Join(outputDir, buildName) + // XXX: support stremaing via statusWriter + _, err = osbuild.RunOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil, nil, false, os.Stderr) + return err +} diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 55c98549ee..a377398bc0 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -19,6 +19,27 @@ type FilterResult struct { ImgType distro.ImageType } +func getOneImage(distroName, imgTypeStr, archStr string) (*FilterResult, error) { + filterExprs := []string{ + fmt.Sprintf("name:%s", distroName), + fmt.Sprintf("arch:%s", archStr), + fmt.Sprintf("type:%s", imgTypeStr), + } + filteredResults, err := getFilteredImages(filterExprs) + if err != nil { + return nil, err + } + switch len(filteredResults) { + case 0: + return nil, fmt.Errorf("cannot find image for %s %s %s", distroName, imgTypeStr, archStr) + case 1: + return &filteredResults[0], nil + default: + return nil, fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr) + } +} + +// XXX: rename FilterDistros to FilterImages(?) func FilterDistros(fac *distrofactory.Factory, distroNames []string, filters Filters) ([]FilterResult, error) { var res []FilterResult diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index dfed369cdf..76ecf695c9 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -39,6 +39,13 @@ func cmdManifest(cmd *cobra.Command, args []string) error { return outputManifest(osStdout, distroName, imgType, archStr) } +func cmdBuild(cmd *cobra.Command, args []string) error { + distroName := args[0] + imgType := args[1] + + return buildImage(osStdout, distroName, imgType) +} + func run() error { // images logs a bunch of stuff to Debug/Info that we we do not // want to show @@ -76,6 +83,16 @@ operating sytsems like centos and RHEL with easy customizations support.`, } rootCmd.AddCommand(manifestCmd) + buildCmd := &cobra.Command{ + Use: "build ", + Short: "Build the given distro/image-type, e.g. centos-9 qcow2", + RunE: cmdBuild, + SilenceUsage: true, + Args: cobra.ExactArgs(2), + } + rootCmd.AddCommand(buildCmd) + // XXX: add --output=text,json and streaming + return rootCmd.Execute() } diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index 5f2c22308a..b2d55a6e54 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -30,29 +30,15 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist } func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error { - filterExprs := []string{ - fmt.Sprintf("name:%s", distroName), - fmt.Sprintf("arch:%s", archStr), - fmt.Sprintf("type:%s", imgTypeStr), - } - filteredResults, err := getFilteredImages(filterExprs) + // XXX: what/how much do we expose here? + var options distro.ImageOptions + + filterResult, err := getOneImage(distroName, imgTypeStr, archStr) if err != nil { return err } - switch len(filteredResults) { - case 0: - return fmt.Errorf("cannot find image for %s %s %s", distroName, imgTypeStr, archStr) - case 1: - // nothing - default: - return fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr) - } - - var bp blueprint.Blueprint - // XXX: what/how much do we expose here? - options := distro.ImageOptions{} - distro := filteredResults[0].Distro - imgType := filteredResults[0].ImgType + distro := filterResult.Distro + imgType := filterResult.ImgType reporeg, err := newRepoRegistry() if err != nil { @@ -62,6 +48,8 @@ func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error if err != nil { return err } + + var bp blueprint.Blueprint preManifest, warnings, err := imgType.Manifest(&bp, options, repos, 0) if err != nil { return err From 4f749fcd41d01d8355d426c6ad5bf5b651c93fa9 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 30 Oct 2024 19:11:04 +0100 Subject: [PATCH 09/17] fixup! move newRepoRegistry() into it's own file --- cmd/image-builder/list_images.go | 22 ---------------------- cmd/image-builder/repos.go | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 cmd/image-builder/repos.go diff --git a/cmd/image-builder/list_images.go b/cmd/image-builder/list_images.go index 18892747ea..44d5d35001 100644 --- a/cmd/image-builder/list_images.go +++ b/cmd/image-builder/list_images.go @@ -2,32 +2,10 @@ package main import ( "io" - "os" - "strings" "github.com/osbuild/images/pkg/distrofactory" - "github.com/osbuild/images/pkg/reporegistry" ) -// XXX: copied from "composer", should be exported there so -// that we keep this in sync -// XXX2: means we need to depend on osbuild-composer-common or something -var repositoryConfigs = []string{ - "/etc/osbuild-composer", - "/usr/share/osbuild-composer", -} - -// XXX: move to pkg/reporegistry -func newRepoRegistry() (*reporegistry.RepoRegistry, error) { - // useful for development/debugging, e.g. run: - // go build && IMAGE_BUILDER_EXTRA_REPOS_PATH=../../test/data ./image-builder - if extraReposPath := os.Getenv("IMAGE_BUILDER_EXTRA_REPOS_PATH"); extraReposPath != "" { - repositoryConfigs = append(repositoryConfigs, strings.Split(extraReposPath, ":")...) - } - - return reporegistry.New(repositoryConfigs) -} - func getFilteredImages(filterExprs []string) ([]FilterResult, error) { repos, err := newRepoRegistry() if err != nil { diff --git a/cmd/image-builder/repos.go b/cmd/image-builder/repos.go new file mode 100644 index 0000000000..b1e38a269b --- /dev/null +++ b/cmd/image-builder/repos.go @@ -0,0 +1,27 @@ +package main + +import ( + "os" + "strings" + + "github.com/osbuild/images/pkg/reporegistry" +) + +// XXX: copied from "composer", should be exported there so +// that we keep this in sync +// XXX2: means we need to depend on osbuild-composer-common or something +var repositoryConfigs = []string{ + "/etc/osbuild-composer", + "/usr/share/osbuild-composer", +} + +// XXX: move this new env into pkg/reporegistry? +func newRepoRegistry() (*reporegistry.RepoRegistry, error) { + // useful for development/debugging, e.g. run: + // go build && IMAGE_BUILDER_EXTRA_REPOS_PATH=../../test/data ./image-builder + if extraReposPath := os.Getenv("IMAGE_BUILDER_EXTRA_REPOS_PATH"); extraReposPath != "" { + repositoryConfigs = append(repositoryConfigs, strings.Split(extraReposPath, ":")...) + } + + return reporegistry.New(repositoryConfigs) +} From ac0dd397c6a405b6bc67eb7b3a821f1b3c62e97d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 6 Nov 2024 17:10:05 +0100 Subject: [PATCH 10/17] fixup! rebase on distrofilter branch --- cmd/image-builder/filters.go | 140 ++++-------------------------- cmd/image-builder/filters_test.go | 62 ------------- cmd/image-builder/list_images.go | 18 +--- 3 files changed, 19 insertions(+), 201 deletions(-) delete mode 100644 cmd/image-builder/filters_test.go diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index a377398bc0..774def7080 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -2,30 +2,32 @@ package main import ( "fmt" - "strings" - "github.com/sirupsen/logrus" - - "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/distrofactory" - "github.com/osbuild/images/pkg/distrosort" + "github.com/osbuild/images/pkg/imagefilter" ) -// XXX: move this file into distro factory? - -type FilterResult struct { - Distro distro.Distro - Arch distro.Arch - ImgType distro.ImageType +func newImageFilterDefault() (*imagefilter.ImageFilter, error) { + fac := distrofactory.NewDefault() + repos, err := newRepoRegistry() + if err != nil { + return nil, err + } + return imagefilter.New(fac, repos) } -func getOneImage(distroName, imgTypeStr, archStr string) (*FilterResult, error) { +func getOneImage(distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) { + imageFilter, err := newImageFilterDefault() + if err != nil { + return nil, err + } + filterExprs := []string{ fmt.Sprintf("name:%s", distroName), fmt.Sprintf("arch:%s", archStr), fmt.Sprintf("type:%s", imgTypeStr), } - filteredResults, err := getFilteredImages(filterExprs) + filteredResults, err := imageFilter.Filter(filterExprs...) if err != nil { return nil, err } @@ -38,115 +40,3 @@ func getOneImage(distroName, imgTypeStr, archStr string) (*FilterResult, error) return nil, fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr) } } - -// XXX: rename FilterDistros to FilterImages(?) -func FilterDistros(fac *distrofactory.Factory, distroNames []string, filters Filters) ([]FilterResult, error) { - var res []FilterResult - - if err := distrosort.Names(distroNames); err != nil { - return nil, err - } - for _, distroName := range distroNames { - distro := fac.GetDistro(distroName) - if distro == nil { - logrus.Debugf("skipping %v: has repositories but unsupported", distroName) - } - for _, archName := range distro.ListArches() { - a, err := distro.GetArch(archName) - if err != nil { - return nil, err - } - for _, imgTypeName := range a.ListImageTypes() { - imgType, err := a.GetImageType(imgTypeName) - if err != nil { - return nil, err - } - if filters.Matches(distro, a, imgType) { - res = append(res, FilterResult{distro, a, imgType}) - } - } - } - } - - return res, nil -} - -// Filters is a way to filter a list of distros -type Filters []filter - -// NewFilters creates a filtering for a list of distros -func NewFilters(sl []string) (Filters, error) { - var filters []filter - for _, s := range sl { - l := strings.SplitN(s, ":", 2) - switch l[0] { - case s: - filters = append(filters, &distroNameFilter{ - filter: l[0], - exact: false, - }) - case "name": - filters = append(filters, &distroNameFilter{l[1], true}) - case "arch": - filters = append(filters, &archFilter{l[1]}) - case "type": - filters = append(filters, &imgTypeFilter{l[1]}) - // mostly here to show how powerful this is - case "bootmode": - filters = append(filters, &bootmodeFilter{l[1]}) - default: - return nil, fmt.Errorf("unsupported filter prefix: %q", l[0]) - } - } - return filters, nil -} - -// Matches returns true if the given (distro,arch,imgType) tuple matches -// the filter expressions -func (fl Filters) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { - matches := true - for _, f := range fl { - matches = matches && f.Matches(distro, arch, imgType) - } - return matches -} - -type filter interface { - Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool -} - -type distroNameFilter struct { - filter string - exact bool -} - -func (d *distroNameFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { - if d.exact { - return distro.Name() == d.filter - } - return strings.Contains(distro.Name(), d.filter) -} - -type archFilter struct { - filter string -} - -func (d *archFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { - return strings.Contains(arch.Name(), d.filter) -} - -type imgTypeFilter struct { - filter string -} - -func (d *imgTypeFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { - return strings.Contains(imgType.Name(), d.filter) -} - -type bootmodeFilter struct { - filter string -} - -func (d *bootmodeFilter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { - return imgType.BootMode().String() == d.filter -} diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go deleted file mode 100644 index 6cb49e7665..0000000000 --- a/cmd/image-builder/filters_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package main_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/osbuild/images/cmd/image-builder" - "github.com/osbuild/images/pkg/distrofactory" -) - -func TestDistroFilter(t *testing.T) { - fac := distrofactory.NewTestDefault() - - for _, tc := range []struct { - searchExpr []string - distro, arch, imgType string - expectsMatch bool - }{ - // simple (fuzzy on name) - {[]string{"foo"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"test-distro-1"}, "test-distro-1", "test_arch3", "qcow2", true}, - {[]string{"test-distro"}, "test-distro-1", "test_arch3", "qcow2", true}, - // name: prefix (exact matches only) - {[]string{"name:bar"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"name:test-distro-1"}, "test-distro-1", "test_arch3", "qcow2", true}, - {[]string{"name:test-distro"}, "test-distro-1", "test_arch3", "qcow2", false}, - // arch: prefix - // XXX: should we be exact there? what do users expect? - {[]string{"arch:amd64"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"arch:test_arch"}, "test-distro-1", "test_arch3", "qcow2", true}, - {[]string{"arch:test_ar"}, "test-distro-1", "test_arch3", "qcow2", true}, - // type: prefix - // XXX: should we be exact there? what do users expect? - {[]string{"type:ami"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", true}, - {[]string{"type:qcow"}, "test-distro-1", "test_arch3", "qcow2", true}, - // bootmode: prefix - {[]string{"bootmode:uefi"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"bootmode:hybrid"}, "test-distro-1", "test_arch3", "qcow2", true}, - // multiple filters are AND - {[]string{"name:test-distro-1", "type:ami"}, "test-distro-1", "test_arch3", "qcow2", false}, - {[]string{"name:test-distro-1", "type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", true}, - {[]string{"name:test-distro-1", "arch:amd64", "type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", false}, - } { - // XXX: it would be nice if TestDistro would support constructing - // like GetDistro("rhel-8.1:i386,amd64:ami,qcow2") instead of - // the current very static setup - di := fac.GetDistro(tc.distro) - require.NotNil(t, di) - ar, err := di.GetArch(tc.arch) - require.NoError(t, err) - im, err := ar.GetImageType(tc.imgType) - require.NoError(t, err) - ff, err := main.NewFilters(tc.searchExpr) - require.NoError(t, err) - - match := ff.Matches(di, ar, im) - assert.Equal(t, tc.expectsMatch, match, tc) - } -} diff --git a/cmd/image-builder/list_images.go b/cmd/image-builder/list_images.go index 44d5d35001..faebd47b5f 100644 --- a/cmd/image-builder/list_images.go +++ b/cmd/image-builder/list_images.go @@ -2,25 +2,15 @@ package main import ( "io" - - "github.com/osbuild/images/pkg/distrofactory" ) -func getFilteredImages(filterExprs []string) ([]FilterResult, error) { - repos, err := newRepoRegistry() - if err != nil { - return nil, err - } - filters, err := NewFilters(filterExprs) +func listImages(out io.Writer, format string, filterExprs []string) error { + imageFilter, err := newImageFilterDefault() if err != nil { - return nil, err + return err } - fac := distrofactory.NewDefault() - return FilterDistros(fac, repos.ListDistros(), filters) -} -func listImages(out io.Writer, format string, filterExprs []string) error { - filteredResult, err := getFilteredImages(filterExprs) + filteredResult, err := imageFilter.Filter(filterExprs...) if err != nil { return err } From 592fa2147d2c73497a96cafd0ac39fc72f438194 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 6 Nov 2024 17:25:01 +0100 Subject: [PATCH 11/17] fixup! allow "{distro,arch,type}:" prefix to build,manifest to make copy/paste friendly --- cmd/image-builder/filters.go | 2 +- cmd/image-builder/main.go | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 774def7080..500b73d77d 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -23,7 +23,7 @@ func getOneImage(distroName, imgTypeStr, archStr string) (*imagefilter.Result, e } filterExprs := []string{ - fmt.Sprintf("name:%s", distroName), + fmt.Sprintf("distro:%s", distroName), fmt.Sprintf("arch:%s", archStr), fmt.Sprintf("type:%s", imgTypeStr), } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 76ecf695c9..b06d92f19d 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -4,6 +4,7 @@ import ( "io" "log" "os" + "strings" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -27,11 +28,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error { } func cmdManifest(cmd *cobra.Command, args []string) error { - distroName := args[0] - imgType := args[1] + // support prefixes to make it easy to copy/paste from list-images + distroName := strings.TrimPrefix(args[0], "distro:") + imgType := strings.TrimPrefix(args[1], "type:") var archStr string if len(args) > 2 { - archStr = args[2] + archStr = strings.TrimPrefix(args[2], "arch:") } else { archStr = arch.Current().String() } @@ -40,8 +42,9 @@ func cmdManifest(cmd *cobra.Command, args []string) error { } func cmdBuild(cmd *cobra.Command, args []string) error { - distroName := args[0] - imgType := args[1] + // support prefixes to make it easy to copy/paste from list-images + distroName := strings.TrimPrefix(args[0], "distro:") + imgType := strings.TrimPrefix(args[1], "type:") return buildImage(osStdout, distroName, imgType) } @@ -78,8 +81,9 @@ operating sytsems like centos and RHEL with easy customizations support.`, Short: "Build manifest for the given distro/image-type, e.g. centos-9 qcow2", RunE: cmdManifest, SilenceUsage: true, - Args: cobra.MinimumNArgs(2), - Hidden: true, + // XXX: show error with available types if only one arg given + Args: cobra.MinimumNArgs(2), + Hidden: true, } rootCmd.AddCommand(manifestCmd) @@ -88,7 +92,8 @@ operating sytsems like centos and RHEL with easy customizations support.`, Short: "Build the given distro/image-type, e.g. centos-9 qcow2", RunE: cmdBuild, SilenceUsage: true, - Args: cobra.ExactArgs(2), + // XXX: show error with available types if only one arg given + Args: cobra.ExactArgs(2), } rootCmd.AddCommand(buildCmd) // XXX: add --output=text,json and streaming From 7953c6d6edd3345100cfbdf7c3d486d0b3e45e62 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 11 Nov 2024 15:26:19 +0100 Subject: [PATCH 12/17] fixup! update to latest imagefilter-fmt branch --- cmd/image-builder/list_images.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/image-builder/list_images.go b/cmd/image-builder/list_images.go index faebd47b5f..9622b007ba 100644 --- a/cmd/image-builder/list_images.go +++ b/cmd/image-builder/list_images.go @@ -2,6 +2,8 @@ package main import ( "io" + + "github.com/osbuild/images/pkg/imagefilter" ) func listImages(out io.Writer, format string, filterExprs []string) error { @@ -15,7 +17,7 @@ func listImages(out io.Writer, format string, filterExprs []string) error { return err } - fmter, err := NewFilteredResultFormatter(format) + fmter, err := imagefilter.NewResultsFormatter(imagefilter.OutputFormat(format)) if err != nil { return err } From b0386d06ac7262d8875c1805f50fdafbeb7c53e2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 12 Nov 2024 16:43:02 +0100 Subject: [PATCH 13/17] imagefilter: add support to filter for `pkg:` prefixes This commit adds support to filter image based on (unresolved) packages. E.g.: ``` $ ./image-builder list-images --filter pkg:gdisk ... rhel-9.6 type:vhd arch:x86_64 ``` Based on an idea from Achilleas-k, many thanks. --- pkg/imagefilter/filter.go | 33 +++++++++++++++++++++++++++++---- pkg/imagefilter/filter_test.go | 3 +++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pkg/imagefilter/filter.go b/pkg/imagefilter/filter.go index 748df4b86c..988778f7af 100644 --- a/pkg/imagefilter/filter.go +++ b/pkg/imagefilter/filter.go @@ -7,6 +7,7 @@ import ( "github.com/gobwas/glob" + "github.com/osbuild/images/pkg/blueprint" "github.com/osbuild/images/pkg/distro" ) @@ -50,7 +51,7 @@ func newFilter(sl ...string) (*filter, error) { } var supportedFilters = []string{ - "", "distro", "arch", "type", "bootmode", + "", "distro", "arch", "type", "bootmode", "pkg", } type term struct { @@ -66,19 +67,20 @@ type filter struct { // Matches returns true if the given (distro,arch,imgType) tuple matches // the filter expressions -func (fl filter) Matches(distro distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { +func (fl filter) Matches(dist distro.Distro, arch distro.Arch, imgType distro.ImageType) bool { m := true + for _, term := range fl.terms { switch term.prefix { case "": // no prefix, do a "fuzzy" search accross the common // things users may want - m1 := term.pattern.Match(distro.Name()) + m1 := term.pattern.Match(dist.Name()) m2 := term.pattern.Match(arch.Name()) m3 := term.pattern.Match(imgType.Name()) m = m && (m1 || m2 || m3) case "distro": - m = m && term.pattern.Match(distro.Name()) + m = m && term.pattern.Match(dist.Name()) case "arch": m = m && term.pattern.Match(arch.Name()) case "type": @@ -86,7 +88,30 @@ func (fl filter) Matches(distro distro.Distro, arch distro.Arch, imgType distro. // mostly here to show how flexible this is case "bootmode": m = m && term.pattern.Match(imgType.BootMode().String()) + case "pkg": + m = m && containsPackages(imgType, term.pattern) } } return m } + +func containsPackages(imgType distro.ImageType, pattern glob.Glob) bool { + var bp blueprint.Blueprint + manifest, _, err := imgType.Manifest(&bp, distro.ImageOptions{}, nil, 0) + if err != nil { + // XXX: some imgTypes like "iot-*", "edge-*" require a ostree + // url to get instanciated so we miss a bunch of types here + return false + } + + for _, pkgSets := range manifest.GetPackageSetChains() { + for _, pkgSet := range pkgSets { + for _, pkg := range pkgSet.Include { + if pattern.Match(pkg) { + return true + } + } + } + } + return false +} diff --git a/pkg/imagefilter/filter_test.go b/pkg/imagefilter/filter_test.go index eb34528971..1886adeef5 100644 --- a/pkg/imagefilter/filter_test.go +++ b/pkg/imagefilter/filter_test.go @@ -40,6 +40,9 @@ func TestImageFilterFilter(t *testing.T) { // bootmode: prefix {[]string{"bootmode:uefi"}, "test-distro-1", "test_arch3", "qcow2", false}, {[]string{"bootmode:hybrid"}, "test-distro-1", "test_arch3", "qcow2", true}, + // pkg: prefix + {[]string{"pkg:rando-no"}, "test-distro-1", "test_arch3", "qcow2", false}, + {[]string{"pkg:dep-package1"}, "test-distro-1", "test_arch3", "qcow2", true}, // multiple filters are AND {[]string{"distro:test-distro-1", "type:ami"}, "test-distro-1", "test_arch3", "qcow2", false}, {[]string{"distro:test-distro-1", "type:qcow2"}, "test-distro-1", "test_arch3", "qcow2", true}, From 454b834dc0be3da2409292bb1bd775395384124b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 13 Nov 2024 12:07:04 +0100 Subject: [PATCH 14/17] distro: add `OutputFilename` option to ImageOptions This commit allows to change the filename when creating a manifest from an image type. This is useful when doing a UI for users, e.g. this will allow us to support: ``` $ image-builder build rhel-9 type:qcow2 --output foo.img ``` (this will also be useful for bootc-image-builder). Well, osbuild itself will still put it under a directory when doing main_cli.py:exports() but that is something orthogonal. --- pkg/distro/distro.go | 9 +++++++++ pkg/distro/distro_test.go | 18 ++++++++++++++++++ pkg/distro/fedora/images.go | 31 +++++++++++++++---------------- pkg/distro/rhel/images.go | 23 ++++++++++------------- 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/pkg/distro/distro.go b/pkg/distro/distro.go index 741f282a96..674918876d 100644 --- a/pkg/distro/distro.go +++ b/pkg/distro/distro.go @@ -130,6 +130,15 @@ type ImageOptions struct { Subscription *subscription.ImageOptions `json:"subscription,omitempty"` Facts *facts.ImageOptions `json:"facts,omitempty"` PartitioningMode disk.PartitioningMode `json:"partitioning-mode,omitempty"` + + OutputFilename string `json:"output_filename"` +} + +func (i *ImageOptions) Filename(imgType ImageType) string { + if i.OutputFilename != "" { + return i.OutputFilename + } + return imgType.Filename() } type BasePartitionTableMap map[string]disk.PartitionTable diff --git a/pkg/distro/distro_test.go b/pkg/distro/distro_test.go index 5257f73eb3..1b1ba196eb 100644 --- a/pkg/distro/distro_test.go +++ b/pkg/distro/distro_test.go @@ -14,6 +14,7 @@ import ( "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/distro/distro_test_common" + "github.com/osbuild/images/pkg/distro/test_distro" "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/rpmmd" @@ -567,3 +568,20 @@ func TestDistro_ManifestFIPSWarning(t *testing.T) { } } } + +func TestDistro_ImageOptionsFilename(t *testing.T) { + imgType := &test_distro.TestImageType{} + + for _, tc := range []struct { + outputFilename string + expectedFilename string + }{ + {"", "test.img"}, + {"foo.img", "foo.img"}, + } { + imgOpts := &distro.ImageOptions{ + OutputFilename: tc.outputFilename, + } + assert.Equal(t, tc.expectedFilename, imgOpts.Filename(imgType)) + } +} diff --git a/pkg/distro/fedora/images.go b/pkg/distro/fedora/images.go index c40c42e943..29622efd3e 100644 --- a/pkg/distro/fedora/images.go +++ b/pkg/distro/fedora/images.go @@ -334,8 +334,7 @@ func diskImage(workload workload.Workload, return nil, err } img.PartitionTable = pt - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -359,8 +358,7 @@ func containerImage(workload workload.Workload, img.Environment = t.environment img.Workload = workload - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -392,8 +390,7 @@ func liveInstallerImage(workload workload.Workload, if err != nil { return nil, err } - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -466,8 +463,7 @@ func imageInstallerImage(workload workload.Workload, if err != nil { return nil, err } - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -513,7 +509,8 @@ func iotCommitImage(workload workload.Workload, img.Workload = workload img.OSTreeParent = parentCommit img.OSVersion = d.osVersion - img.Filename = t.Filename() + img.Filename = options.Filename(t) + img.InstallWeakDeps = false return img, nil @@ -544,7 +541,8 @@ func bootableContainerImage(workload workload.Workload, img.Workload = workload img.OSTreeParent = parentCommit img.OSVersion = d.osVersion - img.Filename = t.Filename() + img.Filename = options.Filename(t) + img.InstallWeakDeps = false img.BootContainer = true img.BootcConfig = &bootc.Config{ @@ -596,7 +594,7 @@ func iotContainerImage(workload workload.Workload, img.OSTreeParent = parentCommit img.OSVersion = d.osVersion img.ExtraContainerPackages = packageSets[containerPkgsKey] - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -664,8 +662,7 @@ func iotInstallerImage(workload workload.Workload, if err != nil { return nil, err } - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -705,8 +702,8 @@ func iotImage(workload workload.Workload, return nil, err } img.PartitionTable = pt + img.Filename = options.Filename(t) - img.Filename = t.Filename() img.Compression = t.compression return img, nil @@ -747,13 +744,15 @@ func iotSimplifiedInstallerImage(workload workload.Workload, } rawImg.PartitionTable = pt - rawImg.Filename = t.Filename() + filename := options.Filename(t) + rawImg.Filename = filename img := image.NewOSTreeSimplifiedInstaller(rawImg, customizations.InstallationDevice) img.ExtraBasePackages = packageSets[installerPkgsKey] // img.Workload = workload img.Platform = t.platform - img.Filename = t.Filename() + img.Filename = filename + if bpFDO := customizations.GetFDO(); bpFDO != nil { img.FDO = fdo.FromBP(*bpFDO) } diff --git a/pkg/distro/rhel/images.go b/pkg/distro/rhel/images.go index 7a20febcc3..436209c267 100644 --- a/pkg/distro/rhel/images.go +++ b/pkg/distro/rhel/images.go @@ -374,8 +374,7 @@ func DiskImage(workload workload.Workload, return nil, err } img.PartitionTable = pt - - img.Filename = t.Filename() + img.Filename = options.Filename(t) img.VPCForceSize = t.DiskImageVPCForceSize @@ -415,7 +414,7 @@ func EdgeCommitImage(workload workload.Workload, img.Workload = workload img.OSTreeParent = parentCommit img.OSVersion = t.Arch().Distro().OsVersion() - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -445,7 +444,7 @@ func EdgeContainerImage(workload workload.Workload, img.OSTreeParent = parentCommit img.OSVersion = t.Arch().Distro().OsVersion() img.ExtraContainerPackages = packageSets[ContainerPkgsKey] - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -523,8 +522,7 @@ func EdgeInstallerImage(workload workload.Workload, img.OSVersion = t.Arch().Distro().OsVersion() img.Release = fmt.Sprintf("%s %s", t.Arch().Distro().Product(), t.Arch().Distro().OsVersion()) img.FIPS = customizations.GetFIPS() - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -564,8 +562,8 @@ func EdgeRawImage(workload workload.Workload, return nil, err } img.PartitionTable = pt + img.Filename = options.Filename(t) - img.Filename = t.Filename() img.Compression = t.Compression return img, nil @@ -607,13 +605,14 @@ func EdgeSimplifiedInstallerImage(workload workload.Workload, } rawImg.PartitionTable = pt - rawImg.Filename = t.Filename() + filename := options.Filename(t) + rawImg.Filename = filename img := image.NewOSTreeSimplifiedInstaller(rawImg, customizations.InstallationDevice) img.ExtraBasePackages = packageSets[InstallerPkgsKey] // img.Workload = workload img.Platform = t.platform - img.Filename = t.Filename() + img.Filename = filename if bpFDO := customizations.GetFDO(); bpFDO != nil { img.FDO = fdo.FromBP(*bpFDO) } @@ -719,8 +718,7 @@ func ImageInstallerImage(workload workload.Workload, img.Product = d.product img.OSVersion = d.osVersion img.Release = fmt.Sprintf("%s %s", d.product, d.osVersion) - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil } @@ -744,8 +742,7 @@ func TarImage(workload workload.Workload, img.Environment = t.Environment img.Workload = workload - - img.Filename = t.Filename() + img.Filename = options.Filename(t) return img, nil From eb5d108f397b61e72bbea9932fee79ae038e5de4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 13 Nov 2024 18:29:58 +0100 Subject: [PATCH 15/17] image-builder: add `ib build --filename foo.img` support --- cmd/image-builder/build.go | 7 +++++-- cmd/image-builder/main.go | 12 +++++++++--- cmd/image-builder/manifest.go | 20 ++++++++++++++------ 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/cmd/image-builder/build.go b/cmd/image-builder/build.go index ea1b081502..038f6c3772 100644 --- a/cmd/image-builder/build.go +++ b/cmd/image-builder/build.go @@ -11,7 +11,7 @@ import ( "github.com/osbuild/images/pkg/osbuild" ) -func buildImage(out io.Writer, distroName, imgTypeStr string) error { +func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) error { // cross arch building is not possible, we would have to download // a pre-populated buildroot (tar,container) with rpm for that archStr := arch.Current().String() @@ -22,7 +22,10 @@ func buildImage(out io.Writer, distroName, imgTypeStr string) error { imgType := filterResult.ImgType var mf bytes.Buffer - if err := outputManifest(&mf, distroName, imgTypeStr, archStr); err != nil { + opts := &genManifestOptions{ + OutputFilename: outputFilename, + } + if err := outputManifest(&mf, distroName, imgTypeStr, archStr, opts); err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index b06d92f19d..3fe5d460cf 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -38,15 +38,19 @@ func cmdManifest(cmd *cobra.Command, args []string) error { archStr = arch.Current().String() } - return outputManifest(osStdout, distroName, imgType, archStr) + return outputManifest(osStdout, distroName, imgType, archStr, nil) } func cmdBuild(cmd *cobra.Command, args []string) error { // support prefixes to make it easy to copy/paste from list-images distroName := strings.TrimPrefix(args[0], "distro:") imgType := strings.TrimPrefix(args[1], "type:") + outputFilename, err := cmd.Flags().GetString("filename") + if err != nil { + return err + } - return buildImage(osStdout, distroName, imgType) + return buildImage(osStdout, distroName, imgType, outputFilename) } func run() error { @@ -95,8 +99,10 @@ operating sytsems like centos and RHEL with easy customizations support.`, // XXX: show error with available types if only one arg given Args: cobra.ExactArgs(2), } - rootCmd.AddCommand(buildCmd) + // XXX: add this for "manifest" too in a nice way + buildCmd.Flags().String("filename", "", "Output as a specific filename") // XXX: add --output=text,json and streaming + rootCmd.AddCommand(buildCmd) return rootCmd.Execute() } diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index b2d55a6e54..b597e09512 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -29,15 +29,20 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist return depsolvedSets, repoSets, nil } -func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error { - // XXX: what/how much do we expose here? - var options distro.ImageOptions +type genManifestOptions struct { + OutputFilename string +} + +func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string, opts *genManifestOptions) error { + if opts == nil { + opts = &genManifestOptions{} + } filterResult, err := getOneImage(distroName, imgTypeStr, archStr) if err != nil { return err } - distro := filterResult.Distro + dist := filterResult.Distro imgType := filterResult.ImgType reporeg, err := newRepoRegistry() @@ -50,7 +55,10 @@ func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error } var bp blueprint.Blueprint - preManifest, warnings, err := imgType.Manifest(&bp, options, repos, 0) + imgOpts := distro.ImageOptions{ + OutputFilename: opts.OutputFilename, + } + preManifest, warnings, err := imgType.Manifest(&bp, imgOpts, repos, 0) if err != nil { return err } @@ -64,7 +72,7 @@ func outputManifest(out io.Writer, distroName, imgTypeStr, archStr string) error if err != nil { return err } - packageSpecs, _, err := depsolve(cacheDir, preManifest.GetPackageSetChains(), distro, archStr) + packageSpecs, _, err := depsolve(cacheDir, preManifest.GetPackageSetChains(), dist, archStr) if err != nil { return err } From 2971ef50765e3fe70c6d1432ab267b8011d7f222 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 18 Nov 2024 09:40:44 +0100 Subject: [PATCH 16/17] fixup! improve getOneImage and add tests --- cmd/image-builder/export_test.go | 3 ++- cmd/image-builder/filters.go | 7 ++++++- cmd/image-builder/filters_test.go | 27 +++++++++++++++++++++++++++ cmd/image-builder/main_test.go | 6 ++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 cmd/image-builder/filters_test.go diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 11134c3fe4..b040384d5f 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -22,5 +22,6 @@ func MockOsStdout(new io.Writer) (restore func()) { } var ( - Run = run + GetOneImage = getOneImage + Run = run ) diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 500b73d77d..9aebf32236 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -22,6 +22,9 @@ func getOneImage(distroName, imgTypeStr, archStr string) (*imagefilter.Result, e return nil, err } + // XXX: validate using "glob.QuoteMeta(distroName) == distroName",... + // here + filterExprs := []string{ fmt.Sprintf("distro:%s", distroName), fmt.Sprintf("arch:%s", archStr), @@ -33,10 +36,12 @@ func getOneImage(distroName, imgTypeStr, archStr string) (*imagefilter.Result, e } switch len(filteredResults) { case 0: - return nil, fmt.Errorf("cannot find image for %s %s %s", distroName, imgTypeStr, archStr) + return nil, fmt.Errorf("cannot find image for: distro:%q type:%q arch:%q", distroName, imgTypeStr, archStr) case 1: return &filteredResults[0], nil default: + // XXX: imagefilter.Result should have a String() method so + // that this output can actually show the results return nil, fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr) } } diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go new file mode 100644 index 0000000000..4ec4385462 --- /dev/null +++ b/cmd/image-builder/filters_test.go @@ -0,0 +1,27 @@ +package main_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/images/cmd/image-builder" +) + +func TestGetOneImageHappy(t *testing.T) { + t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data") + + res, err := main.GetOneImage("centos-9", "qcow2", "x86_64") + require.NoError(t, err) + assert.Equal(t, "centos-9", res.Distro.Name()) + assert.Equal(t, "x86_64", res.Arch.Name()) + assert.Equal(t, "qcow2", res.ImgType.Name()) +} + +func TestGetOneImageSad(t *testing.T) { + t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data") + + _, err := main.GetOneImage("no-distro-meeh", "qcow2", "x86_64") + require.EqualError(t, err, `cannot find image for: distro:"no-distro-meeh" type:"qcow2" arch:"x86_64"`) +} diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 3571cc8ff4..bbd243590f 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -4,11 +4,17 @@ import ( "bytes" "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/osbuild/images/cmd/image-builder" ) +func init() { + // silence logrus by default, it is quite verbose + logrus.SetLevel(logrus.WarnLevel) +} + func TestListImagesSmoke(t *testing.T) { t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data") From 089cf7c4ebd51c7963147a230227371b23f6279e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 15 Nov 2024 16:21:23 +0100 Subject: [PATCH 17/17] wip: progress reporting via jsonseq-monitor --- cmd/image-builder/build.go | 62 +++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/cmd/image-builder/build.go b/cmd/image-builder/build.go index 038f6c3772..85e3783783 100644 --- a/cmd/image-builder/build.go +++ b/cmd/image-builder/build.go @@ -5,12 +5,68 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "github.com/osbuild/images/pkg/arch" - "github.com/osbuild/images/pkg/osbuild" + "github.com/osbuild/images/pkg/osbuildmonitor" ) +// XXX: merge back into images/pkg/osbuild/osbuild-exec.go or +// into osbuildmonitor +func runOSBuild(manifest []byte, store, outputDirectory string, exports, extraEnv []string) error { + rp, wp, err := os.Pipe() + if err != nil { + return fmt.Errorf("cannot create pipe for osbuild: %w", err) + } + defer rp.Close() + defer wp.Close() + + cmd := exec.Command( + "osbuild", + "--store", store, + "--output-directory", outputDirectory, + "--monitor=JSONSeqMonitor", + "--monitor-fd=3", + "-", + ) + for _, export := range exports { + cmd.Args = append(cmd.Args, "--export", export) + } + + cmd.Env = append(os.Environ(), extraEnv...) + cmd.Stdin = bytes.NewBuffer(manifest) + cmd.Stderr = os.Stderr + // we could use "--json" here and would get the build-result + // exported here + cmd.Stdout = nil + cmd.ExtraFiles = []*os.File{wp} + + if err := cmd.Start(); err != nil { + return fmt.Errorf("error starting osbuild: %v", err) + } + wp.Close() + + scanner := osbuildmonitor.NewStatusScanner(rp) + for { + status, err := scanner.Status() + if err != nil { + return err + } + if status == nil { + break + } + // XXX: add progress bar + fmt.Printf("[%s] %s\n", status.Timestamp.Format("2006-01-02 15:04:05"), status.Trace) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("error running osbuild: %w", err) + } + + return nil +} + func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) error { // cross arch building is not possible, we would have to download // a pre-populated buildroot (tar,container) with rpm for that @@ -33,7 +89,5 @@ func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) er outputDir := "." buildName := fmt.Sprintf("%s-%s-%s", distroName, imgTypeStr, archStr) jobOutputDir := filepath.Join(outputDir, buildName) - // XXX: support stremaing via statusWriter - _, err = osbuild.RunOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil, nil, false, os.Stderr) - return err + return runOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil) }