From 0d83cc555661801af0fffd801f71edbbc6332dc7 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 16:49:39 +0100 Subject: [PATCH 1/8] rm dnf-json Remove the dnf-json script from the repository. Instead, the library will depend on osbuild-depsolve-dnf, an osbuild subpackage, for depsolving. --- dnf-json | 389 ------------------------------------------------------- 1 file changed, 389 deletions(-) delete mode 100755 dnf-json diff --git a/dnf-json b/dnf-json deleted file mode 100755 index 8b8b50d3e3..0000000000 --- a/dnf-json +++ /dev/null @@ -1,389 +0,0 @@ -#!/usr/bin/python3 -# pylint: disable=invalid-name - -""" -A JSON-based interface for depsolving using DNF. - -Reads a request through stdin and prints the result to stdout. -In case of error, a structured error is printed to stdout as well. -""" -import json -import os -import sys -import tempfile -from datetime import datetime - -import dnf -import hawkey - - -class Solver(): - - # pylint: disable=too-many-arguments - def __init__(self, repos, module_platform_id, persistdir, cachedir, arch): - self.base = dnf.Base() - - # Enable fastestmirror to ensure we choose the fastest mirrors for - # downloading metadata (when depsolving) and downloading packages. - self.base.conf.fastestmirror = True - - # We use the same cachedir for multiple architectures. Unfortunately, - # this is something that doesn't work well in certain situations - # with zchunk: - # Imagine that we already have cache for arch1. Then, we use dnf-json - # to depsolve for arch2. If ZChunk is enabled and available (that's - # the case for Fedora), dnf will try to download only differences - # between arch1 and arch2 metadata. But, as these are completely - # different, dnf must basically redownload everything. - # For downloding deltas, zchunk uses HTTP range requests. Unfortunately, - # if the mirror doesn't support multi range requests, then zchunk will - # download one small segment per a request. Because we need to update - # the whole metadata (10s of MB), this can be extremely slow in some cases. - # I think that we can come up with a better fix but let's just disable - # zchunk for now. As we are already downloading a lot of data when - # building images, I don't care if we download even more. - self.base.conf.zchunk = False - - # Set the rest of the dnf configuration. - self.base.conf.module_platform_id = module_platform_id - self.base.conf.config_file_path = "/dev/null" - self.base.conf.persistdir = persistdir - self.base.conf.cachedir = cachedir - self.base.conf.substitutions['arch'] = arch - self.base.conf.substitutions['basearch'] = dnf.rpm.basearch(arch) - - for repo in repos: - self.base.repos.add(self._dnfrepo(repo, self.base.conf)) - self.base.fill_sack(load_system_repo=False) - - # pylint: disable=too-many-branches - @staticmethod - def _dnfrepo(desc, parent_conf=None): - """Makes a dnf.repo.Repo out of a JSON repository description""" - - repo = dnf.repo.Repo(desc["id"], parent_conf) - - if "name" in desc: - repo.name = desc["name"] - if "baseurl" in desc: - repo.baseurl = desc["baseurl"] - elif "metalink" in desc: - repo.metalink = desc["metalink"] - elif "mirrorlist" in desc: - repo.mirrorlist = desc["mirrorlist"] - else: - assert False - - if desc.get("ignoressl", False): - repo.sslverify = False - if "sslcacert" in desc: - repo.sslcacert = desc["sslcacert"] - if "sslclientkey" in desc: - repo.sslclientkey = desc["sslclientkey"] - if "sslclientcert" in desc: - repo.sslclientcert = desc["sslclientcert"] - - if "check_gpg" in desc: - repo.gpgcheck = desc["check_gpg"] - if "check_repogpg" in desc: - repo.repo_gpgcheck = desc["check_repogpg"] - if "gpgkey" in desc: - repo.gpgkey = [desc["gpgkey"]] - if "gpgkeys" in desc: - # gpgkeys can contain a full key, or it can be a URL - # dnf expects urls, so write the key to a temporary location and add the file:// - # path to repo.gpgkey - keydir = os.path.join(parent_conf.persistdir, "gpgkeys") - if not os.path.exists(keydir): - os.makedirs(keydir, mode=0o700, exist_ok=True) - - for key in desc["gpgkeys"]: - if key.startswith("-----BEGIN PGP PUBLIC KEY BLOCK-----"): - # Not using with because it needs to be a valid file for the duration. It - # is inside the temporary persistdir so will be cleaned up on exit. - # pylint: disable=consider-using-with - keyfile = tempfile.NamedTemporaryFile(dir=keydir, delete=False) - keyfile.write(key.encode("utf-8")) - repo.gpgkey.append(f"file://{keyfile.name}") - keyfile.close() - else: - repo.gpgkey.append(key) - - # In dnf, the default metadata expiration time is 48 hours. However, - # some repositories never expire the metadata, and others expire it much - # sooner than that. We therefore allow this to be configured. If nothing - # is provided we error on the side of checking if we should invalidate - # the cache. If cache invalidation is not necessary, the overhead of - # checking is in the hundreds of milliseconds. In order to avoid this - # overhead accumulating for API calls that consist of several dnf calls, - # we set the expiration to a short time period, rather than 0. - repo.metadata_expire = desc.get("metadata_expire", "20s") - - # This option if True disables modularization filtering. Effectively - # disabling modularity for given repository. - if "module_hotfixes" in desc: - repo.module_hotfixes = desc["module_hotfixes"] - - return repo - - @staticmethod - def _timestamp_to_rfc3339(timestamp): - return datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%dT%H:%M:%SZ') - - def dump(self): - packages = [] - for package in self.base.sack.query().available(): - packages.append({ - "name": package.name, - "summary": package.summary, - "description": package.description, - "url": package.url, - "repo_id": package.repoid, - "epoch": package.epoch, - "version": package.version, - "release": package.release, - "arch": package.arch, - "buildtime": self._timestamp_to_rfc3339(package.buildtime), - "license": package.license - }) - return packages - - def search(self, args): - """ Perform a search on the available packages - - args contains a "search" dict with parameters to use for searching. - "packages" list of package name globs to search for - "latest" is a boolean that will return only the latest NEVRA instead - of all matching builds in the metadata. - - eg. - - "search": { - "latest": false, - "packages": ["tmux", "vim*", "*ssh*"] - }, - """ - pkg_globs = args.get("packages", []) - - packages = [] - - # NOTE: Build query one piece at a time, don't pass all to filterm at the same - # time. - available = self.base.sack.query().available() - for name in pkg_globs: - # If the package name glob has * in it, use glob. - # If it has *name* use substr - # If it has neither use exact match - if "*" in name: - if name[0] != "*" or name[-1] != "*": - q = available.filter(name__glob=name) - else: - q = available.filter(name__substr=name.replace("*", "")) - else: - q = available.filter(name__eq=name) - - if args.get("latest", False): - q = q.latest() - - for package in q: - packages.append({ - "name": package.name, - "summary": package.summary, - "description": package.description, - "url": package.url, - "repo_id": package.repoid, - "epoch": package.epoch, - "version": package.version, - "release": package.release, - "arch": package.arch, - "buildtime": self._timestamp_to_rfc3339(package.buildtime), - "license": package.license - }) - return packages - - def depsolve(self, transactions): - last_transaction = [] - - for transaction in transactions: - self.base.reset(goal=True) - self.base.sack.reset_excludes() - - self.base.conf.install_weak_deps = transaction.get("install_weak_deps", False) - - # set the packages from the last transaction as installed - for installed_pkg in last_transaction: - self.base.package_install(installed_pkg, strict=True) - - # depsolve the current transaction - self.base.install_specs( - transaction.get("package-specs"), - transaction.get("exclude-specs"), - reponame=transaction.get("repo-ids"), - ) - self.base.resolve() - - # store the current transaction result - last_transaction.clear() - for tsi in self.base.transaction: - # Avoid using the install_set() helper, as it does not guarantee - # a stable order - if tsi.action not in dnf.transaction.FORWARD_ACTIONS: - continue - last_transaction.append(tsi.pkg) - - dependencies = [] - for package in last_transaction: - dependencies.append({ - "name": package.name, - "epoch": package.epoch, - "version": package.version, - "release": package.release, - "arch": package.arch, - "repo_id": package.repoid, - "path": package.relativepath, - "remote_location": package.remote_location(), - "checksum": ( - f"{hawkey.chksum_name(package.chksum[0])}:" - f"{package.chksum[1].hex()}" - ) - }) - - return dependencies - - -def setup_cachedir(request): - arch = request["arch"] - # If dnf-json is run as a service, we don't want users to be able to set the cache - cache_dir = os.environ.get("OVERWRITE_CACHE_DIR", "") - if cache_dir: - cache_dir = os.path.join(cache_dir, arch) - else: - cache_dir = request.get("cachedir", "") - - if not cache_dir: - return "", {"kind": "Error", "reason": "No cache dir set"} - - return cache_dir, None - - -def solve(request, cache_dir): - command = request["command"] - arch = request["arch"] - module_platform_id = request["module_platform_id"] - arguments = request["arguments"] - - transactions = arguments.get("transactions") - with tempfile.TemporaryDirectory() as persistdir: - try: - solver = Solver( - arguments["repos"], - module_platform_id, - persistdir, - cache_dir, - arch - ) - if command == "dump": - result = solver.dump() - elif command == "depsolve": - result = solver.depsolve(transactions) - elif command == "search": - result = solver.search(arguments.get("search", {})) - - except dnf.exceptions.MarkingErrors as e: - printe("error install_specs") - return None, { - "kind": "MarkingErrors", - "reason": f"Error occurred when marking packages for installation: {e}" - } - except dnf.exceptions.DepsolveError as e: - printe("error depsolve") - # collect list of packages for error - pkgs = [] - for t in transactions: - pkgs.extend(t["package-specs"]) - return None, { - "kind": "DepsolveError", - "reason": f"There was a problem depsolving {', '.join(pkgs)}: {e}" - } - except dnf.exceptions.RepoError as e: - return None, { - "kind": "RepoError", - "reason": f"There was a problem reading a repository: {e}" - } - except dnf.exceptions.Error as e: - printe("error repository setup") - return None, { - "kind": type(e).__name__, - "reason": str(e) - } - return result, None - - -def printe(*msg): - print(*msg, file=sys.stderr) - - -def fail(err): - printe(f"{err['kind']}: {err['reason']}") - print(json.dumps(err)) - sys.exit(1) - - -def respond(result): - print(json.dumps(result)) - - -def validate_request(request): - command = request.get("command") - valid_cmds = ("depsolve", "dump", "search") - if command not in valid_cmds: - return { - "kind": "InvalidRequest", - "reason": f"invalid command '{command}': must be one of {', '.join(valid_cmds)}" - } - - if not request.get("arch"): - return { - "kind": "InvalidRequest", - "reason": "no 'arch' specified" - } - - if not request.get("module_platform_id"): - return { - "kind": "InvalidRequest", - "reason": "no 'module_platform_id' specified" - } - arguments = request.get("arguments") - if not arguments: - return { - "kind": "InvalidRequest", - "reason": "empty 'arguments'" - } - - if not arguments.get("repos"): - return { - "kind": "InvalidRequest", - "reason": "no 'repos' specified" - } - - return None - - -def main(): - request = json.load(sys.stdin) - err = validate_request(request) - if err: - fail(err) - - cachedir, err = setup_cachedir(request) - if err: - fail(err) - result, err = solve(request, cachedir) - if err: - fail(err) - else: - respond(result) - - -if __name__ == "__main__": - main() From 05a7e6921bcc98fc2f496dd8a341621e0c99a629 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 16:53:06 +0100 Subject: [PATCH 2/8] rm Makefile The Makefile is outdated and not very useful in its current form. Leaving it there is a bit misleading and can make people think there's something wrong if they try to use it. --- Makefile | 100 ------------------------------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index c8da84a900..0000000000 --- a/Makefile +++ /dev/null @@ -1,100 +0,0 @@ -# -# Maintenance Helpers -# -# This makefile contains targets used for development, as well as helpers to -# aid automatization of maintenance. Unless a target is documented in -# `make help`, it is not supported and is only meant to be used by developers -# to aid their daily development work. -# -# All supported targets honor the `SRCDIR` variable to find the source-tree. -# For most unsupported targets, you are expected to have the source-tree as -# your working directory. To specify a different source-tree, simply override -# the variable via `SRCDIR=` on the commandline. By default, the working -# directory is used for build output, but `BUILDDIR=` allows overriding -# it. -# - -BUILDDIR ?= . -SRCDIR ?= . - -# -# Automatic Variables -# -# This section contains a bunch of automatic variables used all over the place. -# They mostly try to fetch information from the repository sources to avoid -# hard-coding them in this makefile. -# -# Most of the variables here are pre-fetched so they will only ever be -# evaluated once. This, however, means they are always executed regardless of -# which target is run. -# -# VERSION: -# This evaluates the `Version` field of the specfile. Therefore, it will -# be set to the latest version number of this repository without any -# prefix (just a plain number). -# -# COMMIT: -# This evaluates to the latest git commit sha. This will not work if -# the source is not a git checkout. Hence, this variable is not -# pre-fetched but evaluated at time of use. -# - -COMMIT = $(shell (cd "$(SRCDIR)" && git rev-parse HEAD)) - -# -# Generic Targets -# -# The following is a set of generic targets used across the makefile. The -# following targets are defined: -# -# help -# This target prints all supported targets. It is meant as -# documentation of targets we support and might use outside of this -# repository. -# This is also the default target. -# -# $(BUILDDIR)/ -# $(BUILDDIR)/%/ -# This target simply creates the specified directory. It is limited to -# the build-dir as a safety measure. Note that this requires you to use -# a trailing slash after the directory to not mix it up with regular -# files. Lastly, you mostly want this as order-only dependency, since -# timestamps on directories do not affect their content. -# - -.PHONY: help -help: - @echo "make [TARGETS...]" - @echo - @echo "This is the maintenance makefile of osbuild. The following" - @echo "targets are available:" - @echo - @echo " help: Print this usage information." - @echo " man: Generate all man-pages" - -$(BUILDDIR)/: - mkdir -p "$@" - -$(BUILDDIR)/%/: - mkdir -p "$@" - -# -# Maintenance Targets -# -# The following targets are meant for development and repository maintenance. -# They are not supported nor is their use recommended in scripts. -# - -.PHONY: build -build: - - mkdir -p bin - go build -o bin/osbuild-pipeline ./cmd/osbuild-pipeline/ - go build -o bin/osbuild-upload-azure ./cmd/osbuild-upload-azure/ - go build -o bin/osbuild-upload-aws ./cmd/osbuild-upload-aws/ - go build -o bin/osbuild-upload-gcp ./cmd/osbuild-upload-gcp/ - go build -o bin/osbuild-upload-oci ./cmd/osbuild-upload-oci/ - go build -o bin/osbuild-upload-generic-s3 ./cmd/osbuild-upload-generic-s3/ - go build -o bin/osbuild-mock-openid-provider ./cmd/osbuild-mock-openid-provider - go build -o bin/osbuild-service-maintenance ./cmd/osbuild-service-maintenance - go test -c -tags=integration -o bin/osbuild-dnf-json-tests ./cmd/osbuild-dnf-json-tests/main_test.go - go test -c -tags=integration -o bin/osbuild-image-tests ./cmd/osbuild-image-tests/ From e84561d817678646260cb33eaefcf072392a3c6e Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 17:31:43 +0100 Subject: [PATCH 3/8] rm osbuild-pipeline This is duplicated from osbuild-composer and isn't used in any tests or other tooling. --- cmd/osbuild-pipeline/main.go | 259 ----------------------------------- 1 file changed, 259 deletions(-) delete mode 100644 cmd/osbuild-pipeline/main.go diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go deleted file mode 100644 index 5873dd83e3..0000000000 --- a/cmd/osbuild-pipeline/main.go +++ /dev/null @@ -1,259 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "os" - "path" - - "github.com/osbuild/images/internal/common" - "github.com/osbuild/images/pkg/container" - "github.com/osbuild/images/pkg/distro" - "github.com/osbuild/images/pkg/distroregistry" - "github.com/osbuild/images/pkg/dnfjson" - "github.com/osbuild/images/pkg/ostree" - - "github.com/osbuild/images/pkg/blueprint" - "github.com/osbuild/images/pkg/rpmmd" -) - -type repository struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - BaseURL string `json:"baseurl,omitempty"` - Metalink string `json:"metalink,omitempty"` - MirrorList string `json:"mirrorlist,omitempty"` - GPGKey string `json:"gpgkey,omitempty"` - CheckGPG bool `json:"check_gpg,omitempty"` - CheckRepoGPG bool `json:"repo_check_gpg,omitempty"` - IgnoreSSL bool `json:"ignore_ssl,omitempty"` - PackageSets []string `json:"package_sets,omitempty"` - RHSM bool `json:"rhsm,omitempty"` -} - -type ostreeOptions struct { - Ref string `json:"ref"` - Parent string `json:"parent"` - URL string `json:"url"` -} - -type composeRequest struct { - Distro string `json:"distro"` - Arch string `json:"arch"` - ImageType string `json:"image-type"` - Blueprint blueprint.Blueprint `json:"blueprint"` - Repositories []repository `json:"repositories"` - OSTree ostreeOptions `json:"ostree"` -} - -// osbuild-pipeline is a utility command and is often run from within the -// source tree. Find the dnf-json binary in case the osbuild-composer package -// isn't installed. This prioritises the local source version over the system -// version if run from within the source tree. -func findDnfJsonBin() string { - locations := []string{"./dnf-json", "/usr/libexec/osbuild-composer/dnf-json", "/usr/lib/osbuild-composer/dnf-json"} - for _, djPath := range locations { - _, err := os.Stat(djPath) - if !os.IsNotExist(err) { - return djPath - } - } - - // can't run: panic - panic(fmt.Sprintf("could not find 'dnf-json' in any of the known paths: %+v", locations)) -} - -func resolveContainers(sourceSpecs []container.SourceSpec, archName string) ([]container.Spec, error) { - if len(sourceSpecs) == 0 { - return nil, nil - } - - resolver := container.NewResolver(archName) - - for _, c := range sourceSpecs { - resolver.Add(c) - } - - return resolver.Finish() -} - -func main() { - var rpmmdArg bool - flag.BoolVar(&rpmmdArg, "rpmmd", false, "output rpmmd struct instead of pipeline manifest") - var seedArg int64 - flag.Int64Var(&seedArg, "seed", 0, "seed for generating manifests (default: 0)") - flag.Parse() - - // Path to composeRequet or '-' for stdin - composeRequestArg := flag.Arg(0) - - composeRequest := &composeRequest{} - if composeRequestArg != "" { - var reader io.Reader - if composeRequestArg == "-" { - reader = os.Stdin - } else { - var err error - reader, err = os.Open(composeRequestArg) - if err != nil { - panic("Could not open compose request: " + err.Error()) - } - } - file, err := io.ReadAll(reader) - if err != nil { - panic("Could not read compose request: " + err.Error()) - } - err = json.Unmarshal(file, &composeRequest) - if err != nil { - panic("Could not parse blueprint: " + err.Error()) - } - } - - distros := distroregistry.NewDefault() - d := distros.GetDistro(composeRequest.Distro) - if d == nil { - _, _ = fmt.Fprintf(os.Stderr, "The provided distribution '%s' is not supported. Use one of these:\n", composeRequest.Distro) - for _, d := range distros.List() { - _, _ = fmt.Fprintln(os.Stderr, " *", d) - } - return - } - - arch, err := d.GetArch(composeRequest.Arch) - if err != nil { - fmt.Fprintf(os.Stderr, "The provided architecture '%s' is not supported by %s. Use one of these:\n", composeRequest.Arch, d.Name()) - for _, a := range d.ListArches() { - _, _ = fmt.Fprintln(os.Stderr, " *", a) - } - return - } - - imageType, err := arch.GetImageType(composeRequest.ImageType) - if err != nil { - fmt.Fprintf(os.Stderr, "The provided image type '%s' is not supported by %s for %s. Use one of these:\n", composeRequest.ImageType, d.Name(), arch.Name()) - for _, t := range arch.ListImageTypes() { - _, _ = fmt.Fprintln(os.Stderr, " *", t) - } - return - } - - repos := make([]rpmmd.RepoConfig, len(composeRequest.Repositories)) - for idx := range composeRequest.Repositories { - repo := composeRequest.Repositories[idx] - repoName := repo.Name - if repoName == "" { - repoName = fmt.Sprintf("repo-%d", idx) - } - repoId := repo.Id - if repoId == "" { - repoId = fmt.Sprintf("repo-%d", idx) - } - var urls []string - if repo.BaseURL != "" { - urls = []string{repo.BaseURL} - } - var keys []string - if repo.GPGKey != "" { - keys = []string{repo.GPGKey} - } - - repos[idx] = rpmmd.RepoConfig{ - Id: repoId, - Name: repoName, - BaseURLs: urls, - Metalink: repo.Metalink, - MirrorList: repo.MirrorList, - GPGKeys: keys, - CheckGPG: &repo.CheckGPG, - CheckRepoGPG: common.ToPtr(false), - IgnoreSSL: common.ToPtr(false), - PackageSets: repo.PackageSets, - RHSM: repo.RHSM, - } - } - - options := distro.ImageOptions{ - Size: imageType.Size(0), - OSTree: &ostree.ImageOptions{ - ImageRef: composeRequest.OSTree.Ref, - ParentRef: composeRequest.OSTree.Parent, - URL: composeRequest.OSTree.URL, - }, - } - - home, err := os.UserHomeDir() - if err != nil { - panic("os.UserHomeDir(): " + err.Error()) - } - - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd")) - solver.SetDNFJSONPath(findDnfJsonBin()) - - // Set cache size to 3 GiB - // osbuild-pipeline is often used to generate a lot of manifests in a row - // let the cache grow to fit much more repository metadata than we usually allow - solver.SetMaxCacheSize(3 * common.GiB) - - manifest, _, err := imageType.Manifest(&composeRequest.Blueprint, options, repos, seedArg) - if err != nil { - panic(err.Error()) - } - - depsolvedSets := make(map[string][]rpmmd.PackageSpec) - for name, pkgSet := range manifest.GetPackageSetChains() { - res, err := solver.Depsolve(pkgSet) - if err != nil { - panic("Could not depsolve: " + err.Error()) - } - depsolvedSets[name] = res - } - - containerSources := manifest.GetContainerSourceSpecs() - containers := make(map[string][]container.Spec, len(containerSources)) - for name, sourceSpecs := range containerSources { - containerSpecs, err := resolveContainers(sourceSpecs, arch.Name()) - if err != nil { - panic("Could not resolve containers: " + err.Error()) - } - containers[name] = containerSpecs - } - - commitSources := manifest.GetOSTreeSourceSpecs() - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - commitSpec, err := ostree.Resolve(commitSource) - if err != nil { - panic("Could not resolve ostree commit: " + err.Error()) - } - commitSpecs[idx] = commitSpec - } - commits[name] = commitSpecs - } - - var bytes []byte - if rpmmdArg { - bytes, err = json.Marshal(depsolvedSets) - if err != nil { - panic(err) - } - } else { - ms, err := manifest.Serialize(depsolvedSets, containers, commits) - if err != nil { - panic(err.Error()) - } - - bytes, err = json.Marshal(ms) - if err != nil { - panic(err) - } - } - os.Stdout.Write(bytes) - if err := solver.CleanCache(); err != nil { - // print to stderr but don't exit with error - fmt.Fprintf(os.Stderr, "Error during rpm repo cache cleanup: %s", err.Error()) - } -} From 4894a9a75f4d08f81a5406292152d93d2492c1b4 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 21:26:53 +0100 Subject: [PATCH 4/8] github: install osbuild-depsolve-dnf for unit tests Install osbuild-depsolve-dnf to run unit tests. Remove the linter check for dnf-json. --- .github/workflows/tests.yml | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 13358895a1..c1c2e24d6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,10 +21,10 @@ jobs: steps: # krb5-devel is needed to test internal/upload/koji package - # gcc is needed to build the mock dnf-json binary for the unit tests + # gcc is needed to build the mock depsolver binary for the unit tests # gpgme-devel is needed for container upload dependencies - name: Install build and test dependencies - run: dnf -y install krb5-devel gcc git-core go gpgme-devel + run: dnf -y install krb5-devel gcc git-core go gpgme-devel osbuild-depsolve-dnf - name: Check out code into the Go module directory uses: actions/checkout@v4 @@ -37,31 +37,12 @@ jobs: - name: Run unit tests run: go test -race -covermode=atomic -coverprofile=coverage.txt -coverpkg=$(go list ./... | grep -v rpmmd/test$ | tr "\n" ",") ./... - - name: Run dnfjson tests with force-dnf to make sure it's not skipped for any reason + - name: Run depsolver tests with force-dnf to make sure it's not skipped for any reason run: go test -race ./pkg/dnfjson/... -force-dnf - name: Send coverage to codecov.io run: bash <(curl -s https://codecov.io/bash) - python-lint: - name: "🐍 Lint (dnf-json)" - runs-on: ubuntu-latest - container: - image: registry.fedoraproject.org/fedora:latest - steps: - - - name: Install build and test dependencies - run: dnf -y install python3-pylint git-core python3-requests - - - name: Check out code into the Go module directory - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Analysing the code with pylint - run: | - python3 -m pylint dnf-json - lint: name: "⌨ Lint" runs-on: ubuntu-latest From bbc35f4b2260db466abcc7ee6ddc49e2898716a3 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 21:27:50 +0100 Subject: [PATCH 5/8] dnfjson: osbuild-depsolve-dnf finder function Add findDepsolveDnf() to the dnfjson package and call it when initialising the solver. This used to be a helper function in some of the cmds and would look for dnf-json in the current directory for running it from sources then fall back to the installed locations. Now that it's no longer part of the repository, it only searches for the installed locations, `/usr/libexec` for the official Fedora, CentOS, and RHEL paths, and `/usr/lib/osbuild` for other distros that might not user `/usr/libexec`. --- cmd/build/main.go | 1 - cmd/gen-manifests/main.go | 1 - cmd/osbuild-playground/main.go | 17 ----------------- cmd/osbuild-playground/playground.go | 1 - pkg/dnfjson/dnfjson.go | 23 ++++++++++++++++++++--- pkg/dnfjson/dnfjson_test.go | 3 --- 6 files changed, 20 insertions(+), 26 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index e56bd3baa9..16281521cb 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -177,7 +177,6 @@ func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) (map[s func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) - solver.SetDNFJSONPath("./dnf-json") depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, pkgSet := range packageSets { res, err := solver.Depsolve(pkgSet) diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index a36e3b1ec4..fe6c3df658 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -412,7 +412,6 @@ func mockResolveCommits(commitSources map[string][]ostree.SourceSpec) map[string func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) - solver.SetDNFJSONPath("./dnf-json") depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, pkgSet := range packageSets { res, err := solver.Depsolve(pkgSet) diff --git a/cmd/osbuild-playground/main.go b/cmd/osbuild-playground/main.go index 4fe5c65b95..19854832d5 100644 --- a/cmd/osbuild-playground/main.go +++ b/cmd/osbuild-playground/main.go @@ -21,23 +21,6 @@ func AddImageType(img image.ImageKind) { ImageTypes[img.Name()] = img } -// osbuild-playground is a utility command and is often run from within the -// source tree. Find the dnf-json binary in case the osbuild-composer package -// isn't installed. This prioritises the local source version over the system -// version if run from within the source tree. -func findDnfJsonBin() string { - locations := []string{"./dnf-json", "/usr/libexec/osbuild-composer/dnf-json", "/usr/lib/osbuild-composer/dnf-json"} - for _, djPath := range locations { - _, err := os.Stat(djPath) - if !os.IsNotExist(err) { - return djPath - } - } - - // can't run: panic - panic(fmt.Sprintf("could not find 'dnf-json' in any of the known paths: %+v", locations)) -} - func main() { var distroArg string flag.StringVar(&distroArg, "distro", "host", "distro to build from") diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index 3ae2c3996b..44a6b28c39 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -19,7 +19,6 @@ import ( func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos map[string][]rpmmd.RepoConfig, state_dir string) { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(state_dir, "rpmmd")) - solver.SetDNFJSONPath(findDnfJsonBin()) // Set cache size to 1 GiB solver.SetMaxCacheSize(1 * common.GiB) diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index ef8f7f03c3..365f5abc4b 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -37,19 +37,36 @@ type BaseSolver struct { // Cache information cache *rpmCache - // Path to the dnf-json binary and optional args (default: "/usr/libexec/osbuild-composer/dnf-json") + // Path to the dnf-json binary and optional args (default: "/usr/libexec/osbuild-depsolve-dnf") dnfJsonCmd []string resultCache *dnfCache } +// Find the osbuild-depsolve-dnf script. This checks the default location in +// /usr/libexec but also /usr/lib in case it's used on a distribution that +// doesn't use libexec. +func findDepsolveDnf() string { + locations := []string{"/usr/libexec/osbuild-depsolve-dnf", "/usr/lib/osbuild/osbuild-depsolve-dnf"} + for _, djPath := range locations { + _, err := os.Stat(djPath) + if !os.IsNotExist(err) { + return djPath + } + } + + // if it's not found, return empty string; the run() function will fail if + // it's used before setting. + return locations[0] +} + // Create a new unconfigured BaseSolver (without platform information). It can // be used to create configured Solver instances with the NewWithConfig() // method. func NewBaseSolver(cacheDir string) *BaseSolver { return &BaseSolver{ cache: newRPMCache(cacheDir, 1024*1024*1024), // 1 GiB - dnfJsonCmd: []string{"/usr/libexec/osbuild-composer/dnf-json"}, + dnfJsonCmd: []string{findDepsolveDnf()}, resultCache: NewDNFCache(60 * time.Second), } } @@ -620,7 +637,7 @@ func ParseError(data []byte) Error { func run(dnfJsonCmd []string, req *Request) ([]byte, error) { if len(dnfJsonCmd) == 0 { - return nil, fmt.Errorf("dnf-json command undefined") + return nil, fmt.Errorf("osbuild-depsolve-dnf command undefined") } ex := dnfJsonCmd[0] args := make([]string, len(dnfJsonCmd)-1) diff --git a/pkg/dnfjson/dnfjson_test.go b/pkg/dnfjson/dnfjson_test.go index 5247a35958..e4e8efd5a1 100644 --- a/pkg/dnfjson/dnfjson_test.go +++ b/pkg/dnfjson/dnfjson_test.go @@ -42,7 +42,6 @@ func TestDepsolver(t *testing.T) { tmpdir := t.TempDir() solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir) - solver.SetDNFJSONPath("../../dnf-json") { // single depsolve pkgsets := []rpmmd.PackageSet{{Include: []string{"kernel", "vim-minimal", "tmux", "zsh"}, Repositories: []rpmmd.RepoConfig{s.RepoConfig}, InstallWeakDeps: true}} // everything you'll ever need @@ -602,7 +601,6 @@ func TestErrorRepoInfo(t *testing.T) { } solver := NewSolver("f38", "38", "x86_64", "fedora-38", "/tmp/cache") - solver.SetDNFJSONPath("../../dnf-json") for idx, tc := range testCases { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { _, err := solver.Depsolve([]rpmmd.PackageSet{ @@ -632,7 +630,6 @@ func TestRepoConfigHash(t *testing.T) { } solver := NewSolver("f38", "38", "x86_64", "fedora-38", "/tmp/cache") - solver.SetDNFJSONPath("../../dnf-json") rcs, err := solver.reposFromRPMMD(repos) assert.Nil(t, err) From f4309696013ce5be1ea232260e56a98188b3a1f0 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 21:39:54 +0100 Subject: [PATCH 6/8] test: add depsolver in test runners --- test/scripts/configure-generators | 4 ++-- test/scripts/generate-build-config | 2 +- test/scripts/generate-ostree-build-config | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/scripts/configure-generators b/test/scripts/configure-generators index 288b855eac..3568e2fbac 100755 --- a/test/scripts/configure-generators +++ b/test/scripts/configure-generators @@ -32,7 +32,7 @@ generate-build-config-{distro}-{arch}: script: - sudo ./schutzbot/setup-osbuild-repo - sudo dnf -y install go python3 gpgme-devel s3cmd - osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux + osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux osbuild-depsolve-dnf - ./test/scripts/generate-build-config --distro {distro} --arch {arch} build-config.yml artifacts: paths: @@ -65,7 +65,7 @@ generate-ostree-build-config-{distro}-{arch}: script: - sudo ./schutzbot/setup-osbuild-repo - sudo dnf -y install go python3 gpgme-devel s3cmd - osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux podman + osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux osbuild-depsolve-dnf podman - ./test/scripts/generate-ostree-build-config --distro {distro} --arch {arch} build-config.yml build-configs artifacts: paths: diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index d6e81e5812..d3d178f675 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -16,7 +16,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: script: - sudo ./schutzbot/setup-osbuild-repo - sudo dnf install -y go gpgme-devel gcc - osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux + osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux osbuild-depsolve-dnf s3cmd xz - ./test/scripts/build-image "{distro}" "{image_type}" "{config}" - ./test/scripts/boot-image "{distro}" "{arch}" "{image_type}" "{image_path}" diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index 68774e6ba5..6eeae34f86 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -13,7 +13,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: script: - sudo ./schutzbot/setup-osbuild-repo - sudo dnf install -y go gpgme-devel gcc - osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux + osbuild osbuild-luks2 osbuild-lvm2 osbuild-ostree osbuild-selinux osbuild-depsolve-dnf s3cmd podman xz - {start_container} - ./test/scripts/build-image "{distro}" "{image_type}" "{config}" From 5059e52d83d855fe49486e4a6b66b65b04932147 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 21:40:41 +0100 Subject: [PATCH 7/8] devcontainer: add depsolver --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e5369d8f5a..60312e42bf 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -16,6 +16,7 @@ RUN dnf install -y \ osbuild-lvm2 \ osbuild-luks2 \ osbuild-ostree \ + osbuild-depsolve-dnf \ osbuild-tools # install the language server RUN go install -v golang.org/x/tools/gopls@latest From a7ce2ab7e33dcf172b1f878562aa4dae8f005ae8 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 12 Dec 2023 21:41:01 +0100 Subject: [PATCH 8/8] docs: update developer instructions --- docs/developer/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/developer/README.md b/docs/developer/README.md index 7165c903ee..7a5dac17e8 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -2,11 +2,19 @@ ## Local development environment -The build-requirements for Fedora (and other rpm-based distributions) are: -- `gpgme-devel` - +To build most binaries defined in `cmd` and run tests you will need to install `gpgme-devel`. +To generate manifests, you will need to install the `osbuild-depsolve-dnf` package. To build images, you will also need to install `osbuild` and its sub-packages. +The full list of dependencies is: +- `gpgme-devel` +- `osbuild` +- `osbuild-depsolve-dnf` +- `osbuild-luks2` +- `osbuild-lvm2` +- `osbuild-ostree` +- `osbuild-selinux` + ## Topics - [Useful cmds](./cmds.md) for development and testing.