cmd-buildextend-live

#!/usr/bin/env python3
# NOTE: PYTHONUNBUFFERED is set in the entrypoint for unbuffered output
#
# An operation that creates an ISO image for installing CoreOS

import argparse
import hashlib
import json
import os
import re
import shutil
import struct
import subprocess
import sys
import tarfile
import tempfile
import time
import yaml

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from cosalib.builds import Builds
from cosalib.cmdlib import runcmd, sha256sum_file
from cosalib.cmdlib import import_ostree_commit, get_basearch, ensure_glob
from cosalib.meta import GenericBuildMeta

live_exclude_kargs = set([
    '$ignition_firstboot',   # unsubstituted variable in grub config
    'console',               # no serial console by default on ISO
    'ignition.platform.id',  # we hardcode "metal"
    'ostree',                # dracut finds the tree automatically
])

# Parse args and dispatch
parser = argparse.ArgumentParser()
parser.add_argument("--build", help="Build ID")
parser.add_argument("--fast", action='store_true', default=False,
                    help="Reduce compression for development (FCOS only)")
parser.add_argument("--force", action='store_true', default=False,
                    help="Overwrite previously generated installer")
parser.add_argument("--fixture", action='store_true', default=False,
                    help="Create non-functional ISO as coreos-installer test fixture")
args = parser.parse_args()

# Identify the builds and target the latest build if none provided
builds = Builds()
if not args.build:
    args.build = builds.get_latest()
print(f"Targeting build: {args.build}")

srcdir_prefix = "src/config/live/"

if not os.path.isdir(srcdir_prefix):
    raise Exception(f"missing directory {srcdir_prefix}")

workdir = os.path.abspath(os.getcwd())
builddir = builds.get_build_dir(args.build)
buildmeta_path = os.path.join(builddir, 'meta.json')
buildmeta = GenericBuildMeta(workdir=workdir, build=args.build)
repo = os.path.join(workdir, 'tmp/repo')

# Grab the commit hash for this build
buildmeta_commit = buildmeta['ostree-commit']

import_ostree_commit(workdir, builddir, buildmeta)
with open(os.path.join(workdir, 'tmp/image.json')) as f:
    image_json = json.load(f)
squashfs_compression = 'lz4' if args.fast else image_json['squashfs-compression']

base_name = buildmeta['name']
if base_name == "rhcos" and args.fast:
    print("'--fast' requires LZ4 compressed squashfs support in the kernel. This is \
currently only available for FCOS")
    sys.exit(1)

# used to lock
build_semaphore = os.path.join(buildmeta.build_dir, ".live.building")
if os.path.exists(build_semaphore):
    raise Exception(
        f"{build_semaphore} exists: another process is building live")

# Don't run if it's already been done, unless forced
if 'live-iso' in buildmeta['images'] and not args.force:
    print(f"'live' has already been built for {args.build}. Skipping.")
    print("You can force a rebuild with '--force'.")
    sys.exit(0)

basearch = get_basearch()
iso_name = f'{base_name}-{args.build}-live.{basearch}.iso'
name_version = f'{base_name}-{args.build}'
# The short volume ID can only be 32 characters (bytes probably).  We may in the future want
# to shorten this more intelligently, otherwise we truncate the
# version which may impede uniqueness.
volid = name_version[0:32]

tmpdir = os.environ.get("FORCE_TMPDIR", f"{workdir}/tmp/buildpost-live")
if os.path.isdir(tmpdir):
    shutil.rmtree(tmpdir)

tmpisoroot = os.path.join(tmpdir, 'live')
tmpisocoreos = os.path.join(tmpisoroot, 'coreos')
tmpisoimages = os.path.join(tmpisoroot, 'images')
tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot')
tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux')
# contents of initramfs on both PXE and ISO
tmpinitrd_base = os.path.join(tmpdir, 'initrd')
# contents of rootfs image
tmpinitrd_rootfs = os.path.join(tmpdir, 'initrd-rootfs')

for d in (tmpdir, tmpisoroot, tmpisocoreos, tmpisoimages, tmpisoimagespxe,
        tmpisoisolinux, tmpinitrd_base, tmpinitrd_rootfs):
    os.mkdir(d)

# Size of file used to embed an Ignition config within a CPIO.
ignition_img_size = 256 * 1024

# Size of the file used to embed miniso data.
miniso_data_file_size = 16 * 1024


# The kernel requires that uncompressed cpio archives appended to an initrd
# start on a 4-byte boundary.  If there's misalignment, it stops unpacking
# and says:
#
#     Initramfs unpacking failed: invalid magic at start of compressed archive
#
# Append NUL bytes to destf until its size is a multiple of 4 bytes.
#
# https://www.kernel.org/doc/Documentation/early-userspace/buffer-format.txt
# https://github.com/torvalds/linux/blob/47ec5303/init/initramfs.c#L463
def align_initrd_for_uncompressed_append(destf):
    offset = destf.tell()
    if offset % 4:
        destf.write(b'\0' * (4 - offset % 4))


# Return OS features table for features.json, which is read by
# coreos-installer {iso|pxe} customize
def get_os_features():
    features = {
        # coreos-installer >= 0.12.0
        'installer-config': True,
        # coreos/fedora-coreos-config@3edd2f28
        'live-initrd-network': True,
    }

    # coreos-installer >= 0.16.0
    try:
        f = runcmd(['/usr/bin/ostree', 'cat', '--repo', repo, buildmeta_commit,
                    '/usr/share/coreos-installer/example-config.yaml'],
                   capture_output=True).stdout.decode()
        features['installer-config-directives'] = {
            k: True for k in yaml.safe_load(f)
        }
    except subprocess.CalledProcessError as e:
        if e.returncode == 1:
            print('coreos-installer example-config.yaml not found.  Not setting feature.')
        else:
            raise

    return features


# https://www.kernel.org/doc/html/latest/admin-guide/initrd.html#compressed-cpio-images
def mkinitrd_pipe(tmproot, destf, compress=True):
    if not compress:
        align_initrd_for_uncompressed_append(destf)
    files = subprocess.check_output(['find', '.', '-mindepth', '1', '-print0'],
                                cwd=tmproot)
    file_list = files.split(b'\0')
    # If there's a root.squashfs, it _must_ be the first file in the cpio
    # archive, since the dracut 20live module assumes its contents are at
    # a fixed offset in the archive.
    squashfs = b'./root.squashfs'
    if squashfs in file_list:
        file_list.remove(squashfs)
        file_list.insert(0, squashfs)
    cpioproc = subprocess.Popen(['cpio', '-o', '-H', 'newc', '-R', 'root:root',
            '--quiet', '--reproducible', '--force-local', '--null',
            '-D', tmproot], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    if compress:
        gzipargs = ['gzip', '-9']
    else:
        gzipargs = ['cat']
    gzipproc = subprocess.Popen(gzipargs, stdin=cpioproc.stdout, stdout=destf)
    cpioproc.stdin.write(b'\0'.join(file_list))
    cpioproc.stdin.close()
    assert cpioproc.wait() == 0, f"cpio exited with {cpioproc.returncode}"
    assert gzipproc.wait() == 0, f"gzip exited with {gzipproc.returncode}"
    # Fix up padding so the user can append the rootfs afterward
    align_initrd_for_uncompressed_append(destf)


def extend_initrd(initramfs, tmproot, compress=True):
    with open(initramfs, 'ab') as fdst:
        mkinitrd_pipe(tmproot, fdst, compress=compress)


def cp_reflink(src, dest):
    subprocess.check_call(['cp', '--reflink=auto', src, dest])


# Make stream hash for `rdcore stream-hash`
# https://github.com/coreos/coreos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41
def make_stream_hash(src, dest):
    bufsize = 2 * 1024 * 1024
    with open(src, 'rb') as inf:
        with open(dest, 'w') as outf:
            outf.write('stream-hash sha256 {}\n'.format(bufsize))
            while True:
                buf = inf.read(bufsize)
                if not buf:
                    break
                outf.write(hashlib.sha256(buf).hexdigest() + '\n')


def generate_iso():
    # convention for kernel and initramfs names
    kernel_img = 'vmlinuz'
    initrd_img = 'initrd.img'
    # other files
    rootfs_img = 'rootfs.img'
    kargs_file = 'kargs.json'
    igninfo_file = 'igninfo.json'

    tmpisofile = os.path.join(tmpdir, iso_name)

    img_metal_obj = buildmeta.get_artifact_meta("metal", unmerged=True)["images"].get("metal")
    if not img_metal_obj:
        raise Exception("Live image generation requires `metal` image")
    img_metal = os.path.join(builddir, img_metal_obj['path'])
    img_metal_checksum = img_metal_obj['sha256']
    img_metal4k = None
    img_metal4k_checksum = None
    img_metal4k_obj = buildmeta.get_artifact_meta("metal4k", unmerged=True)["images"].get("metal4k")
    if not img_metal4k_obj:
        if not args.fast:
            raise Exception("Live image generation requires `metal4k` image (use --fast to ignore)")
        else:
            print("Missing `metal4k` image; ignoring because of --fast")
    else:
        img_metal4k = os.path.join(builddir, img_metal4k_obj['path'])
        img_metal4k_checksum = img_metal4k_obj['sha256']

    # Find the directory under `/usr/lib/modules/<kver>` where the
    # kernel/initrd live. It will be the 2nd entity output by
    # `ostree ls <commit> /usr/lib/modules`
    process = runcmd(['/usr/bin/ostree', 'ls', '--repo', repo,
                     '--nul-filenames-only', f"{buildmeta_commit}",
                     '/usr/lib/modules'], capture_output=True)
    moduledir = process.stdout.decode().split('\0')[1]

    # copy those files out of the ostree into the iso root dir
    initramfs_img = 'initramfs.img'
    for file in [kernel_img, initramfs_img]:
        runcmd(['/usr/bin/ostree', 'checkout', '--force-copy', '--repo', repo,
                '--user-mode', '--subpath', os.path.join(moduledir, file),
                f"{buildmeta_commit}", tmpisoimagespxe])
        # initramfs isn't world readable by default so let's open up perms
        os.chmod(os.path.join(tmpisoimagespxe, file), 0o644)
        if file == initramfs_img:
            os.rename(
                os.path.join(tmpisoimagespxe, initramfs_img),
                os.path.join(tmpisoimagespxe, initrd_img)
            )

    # Generate initramfs stamp file indicating that this is a live
    # initramfs.  Store the build ID in it.
    stamppath = os.path.join(tmpinitrd_base, 'etc/coreos-live-initramfs')
    os.makedirs(os.path.dirname(stamppath), exist_ok=True)
    with open(stamppath, 'w') as fh:
        fh.write(args.build + '\n')

    # Generate rootfs stamp file with the build ID, indicating that the
    # rootfs has been appended and confirming that initramfs and rootfs are
    # from the same build.
    stamppath = os.path.join(tmpinitrd_rootfs, 'etc/coreos-live-rootfs')
    os.makedirs(os.path.dirname(stamppath), exist_ok=True)
    with open(stamppath, 'w') as fh:
        fh.write(args.build + '\n')

    # Add placeholder for Ignition CPIO file.  This allows an external tool,
    # `coreos-installer iso ignition embed`, to modify an existing ISO image
    # to embed a user's custom Ignition config.  The tool wraps the Ignition
    # config in a cpio.xz and write it directly into this file in the ISO
    # image.  The cpio.xz will be read into the initramfs filesystem at
    # runtime and the Ignition Dracut module will ensure that the config is
    # moved where Ignition will see it. We only handle !s390x here since that's
    # the simple case (where layered initrds are supported). The s390x case is
    # handled lower down
    if basearch != 's390x':
        with open(os.path.join(tmpisoimages, 'ignition.img'), 'wb') as fdst:
            fdst.write(bytes(ignition_img_size))
        igninfo_json = {'file': 'images/ignition.img'}

    # Generate JSON file that lists OS features available to
    # coreos-installer {iso|pxe} customize.  Put it in the initramfs for
    # pxe customize and the ISO for iso customize.
    features = json.dumps(get_os_features(), indent=2, sort_keys=True) + '\n'
    featurespath = os.path.join(tmpinitrd_base, 'etc/coreos/features.json')
    os.makedirs(os.path.dirname(featurespath), exist_ok=True)
    with open(featurespath, 'w') as fh:
        fh.write(features)
    with open(os.path.join(tmpisocoreos, 'features.json'), 'w') as fh:
        fh.write(features)

    # Get PRETTY_NAME
    with tempfile.TemporaryDirectory() as tmpd:
        runcmd(['/usr/bin/ostree', 'checkout', '--repo', repo, '--user-mode',
                '--subpath', "/usr/lib/os-release", buildmeta_commit, tmpd])
        pretty_name = subprocess.check_output(['sh', '-euc', '. ./os-release; echo -n $PRETTY_NAME'],
                                              encoding='utf-8', cwd=tmpd)

    # Add osmet files
    tmp_osmet = os.path.join(tmpinitrd_rootfs, img_metal_obj['path'] + '.osmet')
    fast_arg = []
    if args.fast:
        fast_arg = ['--fast']
    print('Generating osmet file for 512b metal image')
    runcmd(['/usr/lib/coreos-assembler/runvm-coreos-installer', img_metal,
            tmp_osmet, 'pack', 'osmet', '/dev/disk/by-id/virtio-coreos',
            '--description', pretty_name, '--checksum', img_metal_checksum,
            '--output', '/var/tmp/coreos-installer-output'] + fast_arg)
    if img_metal4k_obj:
        assert img_metal4k
        assert img_metal4k_checksum
        tmp_osmet4k = os.path.join(tmpinitrd_rootfs, img_metal4k_obj['path'] + '.osmet')
        print('Generating osmet file for 4k metal image')
        runcmd(['/usr/lib/coreos-assembler/runvm-coreos-installer',
                img_metal4k, tmp_osmet4k, 'pack', 'osmet',
                '/dev/disk/by-id/virtio-coreos', '--description', pretty_name,
                '--checksum', img_metal4k_checksum, '--output',
                '/var/tmp/coreos-installer-output'] + fast_arg)

    # Generate root squashfs
    print(f'Compressing squashfs with {squashfs_compression}')
    # Name must be exactly "root.squashfs" because the 20live dracut module
    # makes assumptions about the length of the name in sysroot.mount
    tmp_squashfs = os.path.join(tmpinitrd_rootfs, 'root.squashfs')
    runcmd(['/usr/lib/coreos-assembler/gf-mksquashfs',
           img_metal, tmp_squashfs, squashfs_compression])

    # Generate rootfs image
    iso_rootfs = os.path.join(tmpisoimagespxe, rootfs_img)
    # The rootfs must be uncompressed because the ISO mounts root.squashfs
    # directly from the middle of the file
    extend_initrd(iso_rootfs, tmpinitrd_rootfs, compress=False)
    # Check that the root.squashfs magic number is in the offset hardcoded
    # in sysroot.mount in 20live/live-generator
    with open(iso_rootfs, 'rb') as fh:
        fh.seek(124)
        if fh.read(4) != b'hsqs':
            raise Exception("root.squashfs not at expected offset in rootfs image")
    pxe_rootfs = os.path.join(tmpdir, rootfs_img)
    # Clone to PXE image
    cp_reflink(iso_rootfs, pxe_rootfs)
    # Save stream hash of rootfs for verifying out-of-band fetches
    os.makedirs(os.path.join(tmpinitrd_base, 'etc'), exist_ok=True)
    make_stream_hash(pxe_rootfs, os.path.join(tmpinitrd_base, 'etc/coreos-live-want-rootfs'))
    # Add common content
    iso_initramfs = os.path.join(tmpisoimagespxe, initrd_img)
    extend_initrd(iso_initramfs, tmpinitrd_base)
    # Clone to PXE image
    pxe_initramfs = os.path.join(tmpdir, initrd_img)
    cp_reflink(iso_initramfs, pxe_initramfs)

    # Read and filter kernel arguments for substituting into ISO bootloader
    result = runcmd(['/usr/lib/coreos-assembler/gf-get-kargs',
                    img_metal], stdout=subprocess.PIPE, text=True)
    kargs_array = [karg for karg in result.stdout.split()
            if karg.split('=')[0] not in live_exclude_kargs]
    kargs_array.append(f"coreos.liveiso={volid}")
    kargs = ' '.join(kargs_array)
    print(f'Substituting ISO kernel arguments: {kargs}')

    kargs_json = {'files': []}
    cmdline = ''
    karg_embed_area_length = 0
    # Grab all the contents from the live dir from the configs
    for srcdir, _, filenames in os.walk(srcdir_prefix):
        dir_suffix = srcdir.replace(srcdir_prefix, '', 1)
        dstdir = os.path.join(tmpisoroot, dir_suffix)
        if not os.path.exists(dstdir):
            os.mkdir(dstdir)
        for filename in filenames:
            # Skip development readmes to avoid confusing users
            if filename == 'README-devel.md':
                continue
            srcfile = os.path.join(srcdir, filename)
            dstfile = os.path.join(dstdir, filename)
            # Assumes all files are text
            with open(srcfile) as fh:
                buf = fh.read()
            newbuf = buf.replace('@@KERNEL-ARGS@@', kargs)
            # if we injected kargs, also check for an embed area
            if buf != newbuf:
                karg_area_start = re.search(r'@@KERNEL-ARGS@@', buf)
                buf = newbuf
                karg_area_end = re.search(r'(#+)# COREOS_KARG_EMBED_AREA\n', buf)
                if karg_area_end is not None:
                    file_kargs = buf[karg_area_start.start():karg_area_end.start()]
                    if len(cmdline) == 0:
                        cmdline = file_kargs
                    elif cmdline != file_kargs:
                        raise Exception(f'Default cmdline is different: "{cmdline}" != "{file_kargs}"')

                    length = karg_area_end.start() + len(karg_area_end[1]) - karg_area_start.start()
                    kargs_json['files'].append({
                        'path': os.path.join(dir_suffix, filename),
                        'offset': karg_area_start.start(),
                        'pad': '#',
                        'end': '\n',
                    })
                    if karg_embed_area_length == 0:
                        karg_embed_area_length = length
                    elif length != karg_embed_area_length:
                        raise Exception(f"Karg embed areas of varying length {kargs_json['files']}")
            with open(dstfile, 'w') as fh:
                fh.write(buf)
            shutil.copystat(srcfile, dstfile)
            print(f'{srcfile} -> {dstfile}')

    if karg_embed_area_length > 0:
        assert (karg_embed_area_length > len(cmdline))
        kargs_json.update(
            size=karg_embed_area_length,
            default=cmdline.strip(),
        )

    # These sections are based on lorax templates
    # see https://github.com/weldr/lorax/tree/master/share/templates.d/99-generic

    # Generate the ISO image. Lots of good info here:
    # https://fedoraproject.org/wiki/User:Pjones/BootableCDsForBIOSAndUEFI
    genisoargs = ['/usr/bin/genisoimage', '-verbose',
                  '-V', volid,
                  '-volset', f"{name_version}",
                  # For  greater portability, consider using both
                  # Joliet and Rock Ridge extensions. Umm, OK :)
                  '-rational-rock', '-J', '-joliet-long']

    # For x86_64 legacy boot (BIOS) booting
    if basearch == "x86_64":
        # Install binaries from syslinux package
        isolinuxfiles = [('/usr/share/syslinux/isolinux.bin', 0o755),
                         ('/usr/share/syslinux/ldlinux.c32',  0o755),
                         ('/usr/share/syslinux/libcom32.c32', 0o755),
                         ('/usr/share/syslinux/libutil.c32',  0o755),
                         ('/usr/share/syslinux/vesamenu.c32', 0o755)]
        for src, mode in isolinuxfiles:
            dst = os.path.join(tmpisoisolinux, os.path.basename(src))
            shutil.copyfile(src, dst)
            os.chmod(dst, mode)

        # for legacy bios boot AKA eltorito boot
        genisoargs += ['-eltorito-boot', 'isolinux/isolinux.bin',
                       '-eltorito-catalog', 'isolinux/boot.cat',
                       '-no-emul-boot',
                       '-boot-load-size', '4',
                       '-boot-info-table']

    elif basearch == "ppc64le":
        os.makedirs(os.path.join(tmpisoroot, 'boot/grub'))
        # can be EFI/fedora or EFI/redhat
        grubpath = ensure_glob(os.path.join(tmpisoroot, 'EFI/*/grub.cfg'))
        if len(grubpath) != 1:
            raise Exception(f'Found != 1 grub.cfg files: {grubpath}')
        shutil.move(grubpath[0], os.path.join(tmpisoroot, 'boot/grub/grub.cfg'))
        for f in kargs_json['files']:
            if re.match('^EFI/.*/grub.cfg$', f['path']):
                f['path'] = 'boot/grub/grub.cfg'

        # safely remove things we don't need in the final ISO tree
        for d in ['EFI', 'isolinux', 'zipl.prm']:
            runcmd(['rm', '-rf', os.path.join(tmpisoroot, d)])

        # grub2-mkrescue is a wrapper around xorriso
        genisoargs = ['grub2-mkrescue', '-volid', volid]
    elif basearch == "s390x":
        # Reserve 32MB for the kernel, starting memory address of the initramfs
        # See https://github.com/weldr/lorax/blob/master/share/templates.d/99-generic/s390.tmpl
        INITRD_ADDRESS = '0x02000000'
        lorax_templates = '/usr/share/lorax/templates.d/99-generic/config_files/s390'
        shutil.copy(os.path.join(lorax_templates, 'redhat.exec'), tmpisoimages)
        with open(os.path.join(lorax_templates, 'generic.ins'), 'r') as fp1:
            with open(os.path.join(tmpisoroot, 'generic.ins'), 'w') as fp2:
                [fp2.write(line.replace('@INITRD_LOAD_ADDRESS@', INITRD_ADDRESS)) for line in fp1]
        for prmfile in ['cdboot.prm', 'genericdvd.prm', 'generic.prm']:
            with open(os.path.join(tmpisoimages, prmfile), 'w') as fp1:
                with open(os.path.join(tmpisoroot, 'zipl.prm'), 'r') as fp2:
                    fp1.write(fp2.read().strip())

        # s390x's z/VM CMS files are limited to 8 char for filenames and extensions
        # Also it is nice to keep naming convetion with Fedora/RHEL for existing users and code
        kernel_dest = os.path.join(tmpisoimagespxe, 'kernel.img')
        shutil.move(os.path.join(tmpisoimagespxe, kernel_img), kernel_dest)
        kernel_img = 'kernel.img'

        if args.fixture:
            # truncate it to 128k so it includes the offsets to the initrd and kargs
            # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L24
            with open(kernel_dest, 'rb+') as f:
                f.truncate(128 * 1024)
            with open(iso_initramfs, 'rb+') as f:
                f.truncate(1024)

        # On s390x, we reserve space for the Ignition config in the initrd
        # image directly since the bootloader doesn't support multiple initrds.
        # We do this by inflating the initramfs just for the duration of the
        # `mk-s390image` call.
        initramfs_size = os.stat(iso_initramfs).st_size
        # sanity-check it's 4-byte aligned (see align_initrd_for_uncompressed_append)
        assert initramfs_size % 4 == 0

        # combine kernel, initramfs and cmdline using the mk-s390image tool
        os.truncate(iso_initramfs, initramfs_size + ignition_img_size)
        runcmd(['/usr/bin/mk-s390image',
               kernel_dest,
               os.path.join(tmpisoimages, 'cdboot.img'),
               '-r', iso_initramfs,
               '-p', os.path.join(tmpisoimages, 'cdboot.prm')])
        os.truncate(iso_initramfs, initramfs_size)

        # Get the kargs and initramfs offsets in the cdboot.img. For more info, see:
        # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L23
        CDBOOT_IMG_OFFS_INITRD_START_BYTES = 66568
        CDBOOT_IMG_OFFS_KARGS_START_BYTES = 66688
        CDBOOT_IMG_OFFS_KARGS_MAX_SIZE = 896
        with open(os.path.join(tmpisoimages, 'cdboot.img'), 'rb') as f:
            f.seek(CDBOOT_IMG_OFFS_INITRD_START_BYTES)
            offset = struct.unpack(">Q", f.read(8))[0]

            # sanity-check we're at the right spot by comparing a few bytes
            f.seek(offset)
            with open(iso_initramfs, 'rb') as canonical:
                if f.read(1024) != canonical.read(1024):
                    raise Exception(f"expected initrd at offset {offset}")

            igninfo_json = {
                'file': 'images/cdboot.img',
                'offset': offset + initramfs_size,
                'length': ignition_img_size,
            }

            # kargs are part of 'images/cdboot.img' blob
            kargs_json['files'].append({
                'path': 'images/cdboot.img',
                'offset': CDBOOT_IMG_OFFS_KARGS_START_BYTES,
                'pad': '\0',
                'end': '\0',
            })
            kargs_json.update(
                size=CDBOOT_IMG_OFFS_KARGS_MAX_SIZE,
            )
        # generate .addrsize file for LPAR
        with open(os.path.join(tmpisoimages, 'initrd.addrsize'), 'wb') as addrsize:
            addrsize_data = struct.pack(">iiii", 0, int(INITRD_ADDRESS, 16), 0,
                                        os.stat(iso_initramfs).st_size)
            addrsize.write(addrsize_data)

        # safely remove things we don't need in the final ISO tree
        for d in ['EFI', 'isolinux', 'zipl.prm']:
            runcmd(['rm', '-rf', os.path.join(tmpisoroot, d)])

        genisoargs = ['/usr/bin/xorrisofs', '-verbose',
                      '-volid', volid,
                      '-volset', f"{name_version}",
                      '-rational-rock', '-J', '-joliet-long',
                      '-no-emul-boot', '-eltorito-boot',
                      os.path.join(os.path.relpath(tmpisoimages, tmpisoroot), 'cdboot.img')]

    # For x86_64 and aarch64 UEFI booting
    if basearch in ("x86_64", "aarch64"):
        # Create the efiboot.img file. This is a fat32 formatted
        # filesystem that contains all the files needed for EFI boot
        # from an ISO.
        with tempfile.TemporaryDirectory():

            # In restrictive environments, setgid, setuid and ownership changes
            # may be restricted. This sets the file ownership to root and
            # removes the setgid and setuid bits in the tarball.
            def strip(tarinfo):
                tarinfo.uid = 0
                tarinfo.gid = 0
                if tarinfo.isdir():
                    tarinfo.mode = 0o755
                elif tarinfo.isfile():
                    tarinfo.mode = 0o0644
                return tarinfo

            tmpimageefidir = os.path.join(tmpdir, "efi")
            runcmd(['/usr/bin/ostree', 'checkout', '--repo', repo,
                   '--user-mode', '--subpath',
                   "/usr/lib/bootupd/updates/EFI",
                   buildmeta_commit, tmpimageefidir])

            # Find name of vendor directory
            vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"]
            if len(vendor_ids) != 1:
                raise Exception(f"did not find exactly one EFI vendor ID: {vendor_ids}")
            vendor_id = vendor_ids[0]

            # Always replace live/EFI/{vendor} to actual live/EFI/{vendor_id}
            # https://github.com/openshift/os/issues/954
            dfd = os.open(tmpisoroot, os.O_RDONLY)
            grubfilepath = ensure_glob('EFI/*/grub.cfg', dir_fd=dfd)
            if len(grubfilepath) != 1:
                raise Exception(f'Found != 1 grub.cfg files: {grubfilepath}')
            srcpath = os.path.dirname(grubfilepath[0])
            if srcpath != f'EFI/{vendor_id}':
                print(f"Renaming '{srcpath}' to 'EFI/{vendor_id}'")
                os.rename(srcpath, f"EFI/{vendor_id}", src_dir_fd=dfd, dst_dir_fd=dfd)
                # And update kargs.json
                for file in kargs_json['files']:
                    if file['path'] == grubfilepath[0]:
                        file['path'] = f'EFI/{vendor_id}/grub.cfg'
            os.close(dfd)

            # Delete fallback and its CSV file.  Its purpose is to create
            # EFI boot variables, which we don't want when booting from
            # removable media.
            #
            # A future shim release will merge fallback.efi into the main
            # shim binary and enable the fallback behavior when the CSV
            # exists.  But for now, fail if fallback.efi is missing.
            for path in ensure_glob(os.path.join(tmpimageefidir, "BOOT", "fb*.efi")):
                os.unlink(path)
            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "BOOT*.CSV")):
                os.unlink(path)

            # Drop vendor copies of shim; we already have it in BOOT*.EFI in
            # BOOT
            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "shim*.efi")):
                os.unlink(path)

            # Consolidate remaining files into BOOT.  shim needs GRUB to be
            # there, and the rest doesn't hurt.
            for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "*")):
                shutil.move(path, os.path.join(tmpimageefidir, "BOOT"))
            os.rmdir(os.path.join(tmpimageefidir, vendor_id))

            # Inject a stub grub.cfg pointing to the one in the main ISO image.
            #
            # When booting via El Torito, this stub is not used; GRUB reads
            # the ISO image directly using its own ISO support.  This
            # happens when booting from a CD device, or when the ISO is
            # copied to a USB stick and booted on EFI firmware which prefers
            # to boot a hard disk from an El Torito image if it has one.
            # EDK II in QEMU behaves this way.
            #
            # This stub is used with EFI firmware which prefers to boot a
            # hard disk from an ESP, or which cannot boot a hard disk via El
            # Torito at all.  In that case, GRUB thinks it booted from a
            # partition of the disk (a fake ESP created by isohybrid,
            # pointing to efiboot.img) and needs a grub.cfg there.
            with open(os.path.join(tmpimageefidir, "BOOT", "grub.cfg"), "w") as fh:
                fh.write(f'''search --label "{volid}" --set root --no-floppy
set prefix=($root)/EFI/{vendor_id}
echo "Booting via ESP..."
configfile $prefix/grub.cfg
boot
''')

            # Install binaries from boot partition
            # Manually construct the tarball to ensure proper permissions and ownership
            efitarfile = tempfile.NamedTemporaryFile(suffix=".tar")
            with tarfile.open(efitarfile.name, "w:", dereference=True) as tar:
                tar.add(tmpimageefidir, arcname="/EFI", filter=strip)

            # Create the efiboot.img file (a fat filesystem) in the images/ dir
            # Note: virt-make-fs lets us do this as non-root
            efibootfile = os.path.join(tmpisoimages, 'efiboot.img')
            os.environ["LIBGUESTFS_BACKEND"] = "direct"
            # On RHEL 8, when booting from a disk device (rather than a CD),
            # https://github.com/systemd/systemd/issues/14408 causes the
            # hybrid ESP to race with the ISO9660 filesystem for the
            # /dev/disk/by-label symlink unless the ESP has its own label,
            # so set EFI-SYSTEM for consistency with the metal image.
            # This should not be needed on Fedora or RHEL 9, but seems like
            # a good thing to do anyway.
            runcmd(['virt-make-fs', '--type=vfat', '--label=EFI-SYSTEM',
                   efitarfile.name, efibootfile])

        genisoargs += ['-eltorito-alt-boot',
                       '-efi-boot', 'images/efiboot.img',
                       '-no-emul-boot']

    # We've done everything that might affect kargs, so filter out any files
    # that no longer exist and write out the kargs JSON if it lists any files
    kargs_json['files'] = [f for f in kargs_json['files']
            if os.path.exists(os.path.join(tmpisoroot, f['path']))]
    kargs_json['files'].sort(key=lambda f: f['path'])
    if kargs_json['files']:
        # Store the location of "karg embed areas" for use by
        # `coreos-installer iso kargs modify`
        with open(os.path.join(tmpisocoreos, kargs_file), 'w') as fh:
            json.dump(kargs_json, fh, indent=2, sort_keys=True)
            fh.write('\n')

    # Write out the igninfo.json file. This is used by coreos-installer to know
    # how to embed the Ignition config.
    with open(os.path.join(tmpisocoreos, igninfo_file), 'w') as fh:
        json.dump(igninfo_json, fh, indent=2, sort_keys=True)  # pylint: disable=E0601
        fh.write('\n')

    # Define inputs and outputs
    genisoargs_final = genisoargs + ['-o', tmpisofile, tmpisoroot]

    miniso_data = os.path.join(tmpisocoreos, "miniso.dat")
    with open(miniso_data, 'wb') as f:
        f.truncate(miniso_data_file_size)

    if args.fixture:
        # Replace or delete anything irrelevant to coreos-installer
        with open(os.path.join(tmpisoimages, 'efiboot.img'), 'w') as fh:
            fh.write('efiboot.img\n')
        with open(os.path.join(tmpisoimagespxe, 'rootfs.img'), 'w') as fh:
            fh.write('rootfs data\n')
        with open(os.path.join(tmpisoimagespxe, 'initrd.img'), 'w') as fh:
            fh.write('initrd data\n')
        with open(os.path.join(tmpisoimagespxe, 'vmlinuz'), 'w') as fh:
            fh.write('the kernel\n')
        # this directory doesn't exist on s390x
        if os.path.isdir(tmpisoisolinux):
            with open(os.path.join(tmpisoisolinux, 'isolinux.bin'), 'rb+') as fh:
                flen = fh.seek(0, 2)
                fh.truncate(0)
                fh.truncate(flen)
                fh.seek(64)
                # isohybrid checks for this magic
                fh.write(b'\xfb\xc0\x78\x70')
            for f in ensure_glob(os.path.join(tmpisoisolinux, '*.c32')):
                os.unlink(f)
            for f in ensure_glob(os.path.join(tmpisoisolinux, '*.msg')):
                os.unlink(f)

    runcmd(genisoargs_final)

    # Add MBR, and GPT with ESP, for x86_64 BIOS/UEFI boot when ISO is
    # copied to a USB stick
    if basearch == "x86_64":
        runcmd(['/usr/bin/isohybrid', '--uefi', tmpisofile])

    genisoargs_minimal = genisoargs + ['-o', f'{tmpisofile}.minimal', tmpisoroot]
    # The only difference with the miniso is that we drop these two files.
    # Keep everything else the same to maximize file matching between the
    # two versions so we can get the smallest delta. E.g. we keep the
    # `coreos.liveiso` karg, even though the miniso doesn't need it.
    # coreos-installer takes care of removing it.
    os.unlink(iso_rootfs)
    os.unlink(miniso_data)
    runcmd(genisoargs_minimal)
    if basearch == "x86_64":
        runcmd(['/usr/bin/isohybrid', '--uefi', f'{tmpisofile}.minimal'])
    # this consumes the minimal image
    runcmd(['/usr/lib/coreos-assembler/runvm-coreos-installer', img_metal, '',
            'pack', 'minimal-iso', tmpisofile, f'{tmpisofile}.minimal',
            '--consume'])

    buildmeta['images'].update({
        'live-iso': {
            'path': iso_name,
            'sha256': sha256sum_file(tmpisofile),
            'skip-compression': True,
        }
    })
    shutil.move(tmpisofile, f"{builddir}/{iso_name}")

    kernel_name = f'{base_name}-{args.build}-live-kernel-{basearch}'
    initramfs_name = f'{base_name}-{args.build}-live-initramfs.{basearch}.img'
    rootfs_name = f'{base_name}-{args.build}-live-rootfs.{basearch}.img'
    kernel_file = os.path.join(builddir, kernel_name)
    initramfs_file = os.path.join(builddir, initramfs_name)
    rootfs_file = os.path.join(builddir, rootfs_name)
    shutil.copyfile(os.path.join(tmpisoimagespxe, kernel_img), kernel_file)
    shutil.move(pxe_initramfs, initramfs_file)
    shutil.move(pxe_rootfs, rootfs_file)
    buildmeta['images'].update({
        'live-kernel': {
            'path': kernel_name,
            'sha256': sha256sum_file(kernel_file),
            'skip-compression': True,
        },
        'live-initramfs': {
            'path': initramfs_name,
            'sha256': sha256sum_file(initramfs_file),
            'skip-compression': True,
        },
        'live-rootfs': {
            'path': rootfs_name,
            'sha256': sha256sum_file(rootfs_file),
            'skip-compression': True,
        }
    })

    buildmeta.write(artifact_name='live')
    print(f"Updated: {buildmeta_path}")


# lock and build
with open(build_semaphore, 'w') as f:
    f.write(f"{time.time_ns()}")

try:
    generate_iso()
finally:
    if os.path.exists(build_semaphore):
        os.unlink(build_semaphore)