test: add test of customizations against reference images

This commit extends the testing of otk files to also support
checking that our customizations are correctly covered. This
is done via a new reference image type with full customizations.

For all refrence images that have this full customizatons we
generate a:
```yaml
otk.define:
 user:
  modifications:
   locale: nl_NL.UTF-8
   ...
```
fragment and compare out customizations with the ones that `images`
generated.

test/data/images-ref/README

@@ -4,14 +4,16 @@ identical to images produced by the previous `images` implementation.
 
 Generate new ones with:
 ```
-./gen-image-def <images-git-checkout> <distro_name> <distro_ver> <arch> <type>
+./gen-image-def <images-git-checkout> <distro_name> <distro_ver> <arch> <type> <empty|full>
 ```
 and put them into git. e.g.
 ```
-$ ./gen-image-def ~/devel/osbuild/images centos 9 x86_64 qcow2
-$ git add centos/9/x86_64/qcow2/centos_9-x86_64-qcow2-empty.yaml
+$ ./gen-image-def ~/devel/osbuild/images centos 9 x86_64 qcow2 empty
+$ git add centos/9/x86_64/qcow2/empty/centos_9-x86_64-qcow2-empty.yaml
 ```
 (ideally we would generate them on the fly).
 
 
+When doing customizations, pass "full" as the last parameter.
+
 [0] https://github.com/osbuild/images

test/data/images-ref/gen-image-def

@@ -11,6 +11,20 @@ import tempfile
 import yaml
 
 
+# XXX: keep in syncwith "test_against_images_ref.py"
+# note that the names of customizations/modifications may not fully match
+full_customizations = {
+    "name": "full",
+    "blueprint": {
+        "customizations": {
+            "locale": {
+                "languages": ["nl_NL.UTF-8"],
+            },
+        },
+    },
+}
+
+
 # see https://github.com/yaml/pyyaml/issues/234#issuecomment-765894586
 class Dumper(yaml.Dumper):  # pylint: disable=too-many-ancestors
     def increase_indent(self, flow: bool = False, indentless: bool = False) -> None:
@@ -27,10 +41,23 @@ def uuid_for_path(path, osbuild_manifest):
     return ""
 
 
-def generate_reference_image(images_base_dir: str, distro_name: str, distro_ver: str, arch: str, img_type: str) -> None:
+def make_buildconfig(customizations: str) -> dict:
+    match customizations:
+        case "empty":
+            return {"name": "empty"}
+        case "full":
+            return full_customizations
+        case _:
+            raise ValueError(f"unknown customization {customizations}")
+
+
+# pylint: disable=too-many-arguments
+def generate_reference_image(images_base_dir: str, distro_name: str,
+                             distro_ver: str, arch: str, img_type: str,
+                             customizations: str) -> None:
     # pylint: disable=consider-using-with
     empty_config = tempfile.NamedTemporaryFile()
-    empty_config.write(b'{"name": "empty"}')
+    empty_config.write(json.dumps(make_buildconfig(customizations)).encode())
     empty_config.flush()
 
     # ideally we would just generate all manifests on the fly but today
@@ -40,7 +67,7 @@ def generate_reference_image(images_base_dir: str, distro_name: str, distro_ver:
     env = os.environ.copy()
     env["OSBUILD_TESTING_RNG_SEED"] = "0"
     distro = f"{distro_name}-{distro_ver}"
-    manifest_path = pathlib.Path.cwd() / distro_name / distro_ver / arch / img_type
+    manifest_path = pathlib.Path.cwd() / distro_name / distro_ver / arch / img_type / customizations
     manifest_path.parent.mkdir(parents=True, exist_ok=True)
     subprocess.check_call([
         "cmd/gen-manifests/gen-manifests",
@@ -52,7 +79,7 @@ def generate_reference_image(images_base_dir: str, distro_name: str, distro_ver:
         "-types", img_type,
         "-config", empty_config.name,
     ], cwd=images_base_dir, env=env)
-    generated = list(manifest_path.glob("*.json"))
+    generated = list(manifest_path.glob(f"*-{customizations}.json"))
     if len(generated) != 1:
         raise ValueError(f"unexpected number of generated manifests: {generated}")
     with open(generated[0], encoding="utf8") as fp:
@@ -78,13 +105,13 @@ def generate_reference_image(images_base_dir: str, distro_name: str, distro_ver:
 
 def run():
     # TODO: add argparse
-    if len(sys.argv) < 5:
+    if len(sys.argv) < 6:
         # TODO: remove images-git-checkout parameter and use something
         # like "go run github.com/osbuild/images/cmd/build@latest" but
         # this needs something like https://github.com/osbuild/images/pull/902
-        print("need <image-git-checkout> <distoname> <distrover> <arch> <img_type> argument", file=sys.stderr)
+        print("need <image-git-checkout> <distoname> <distrover> <arch> <img_type> <customizations>", file=sys.stderr)
         sys.exit(1)
-    generate_reference_image(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])
+    generate_reference_image(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6])
 
 
 if __name__ == "__main__":

test/test_against_images_refs.py

@@ -1,6 +1,7 @@
 import json
 import os
 import pathlib
+import textwrap
 import yaml
 
 import pytest
@@ -14,21 +15,22 @@ class _TestCase:
     def __init__(self, base, ref_yaml_path):
         self.ref_yaml_path = ref_yaml_path
         rel = self.ref_yaml_path.relative_to(base)
-        # dir structure is <distro_name>/<disro_ver>/<arch>/<type>
-        self.distro_name, self.distro_ver, self.arch, self.img_type, _ = rel.as_posix().split("/")
+        # dir structure is <distro_name>/<disro_ver>/<arch>/<type>/<customizations>
+        self.distro_name, self.distro_ver, self.arch, self.img_type, self.customizations, _ = rel.as_posix().split("/")
 
     def as_example_yaml(self):
         # keep in sync with our "example" folder
-        return f"example/{self.distro_name}/{self}.yaml"
+        p = pathlib.Path(__file__).parent.parent
+        return p / f"example/{self.distro_name}/{self.distro_name}-{self.distro_ver}-{self.arch}-{self.img_type}.yaml"
 
     def __str__(self):
-        return f"{self.distro_name}-{self.distro_ver}-{self.arch}-{self.img_type}"
+        return f"{self.distro_name}-{self.distro_ver}-{self.arch}-{self.img_type}-{self.customizations}"
 
 
-def reference_manifests():
+def reference_manifests(customizations: str) -> list:
     tc = []
     base = TEST_DATA_PATH / "images-ref"
-    for path in base.glob("*/*/*/*/*.yaml"):
+    for path in base.glob(f"*/*/*/*/{customizations}/*.yaml"):
         tc.append(_TestCase(base, path))
     return tc
 
@@ -77,8 +79,8 @@ def test_normalize_rpm_refs():
         {"id": "sha256:111"}, {"id": "sha256:222"}]
 
 
-@pytest.mark.parametrize("tc", reference_manifests())
-def test_images_ref(tmp_path, monkeypatch, tc):
+@pytest.mark.parametrize("tc", reference_manifests("empty"))
+def test_images_ref_no_customizations(tmp_path, monkeypatch, tc):
     monkeypatch.setenv("OSBUILD_TESTING_RNG_SEED", "0")
     monkeypatch.setenv("OTK_EXTERNAL_PATH", "./external")
     monkeypatch.setenv("OTK_UNDER_TEST", "1")
@@ -90,7 +92,40 @@ def test_images_ref(tmp_path, monkeypatch, tc):
     otk_json = tmp_path / "manifest-otk.json"
     run(["compile",
          "-o", os.fspath(otk_json),
-         tc.as_example_yaml(),
+         os.fspath(tc.as_example_yaml()),
+         ])
+    with otk_json.open() as fp:
+        manifest = json.load(fp)
+        normalize_rpm_refs(manifest)
+
+    assert manifest == ref_manifest
+
+
+@pytest.mark.parametrize("tc", reference_manifests("full"))
+def test_images_ref_full_customizations(tmp_path, monkeypatch, tc):
+    monkeypatch.setenv("OSBUILD_TESTING_RNG_SEED", "0")
+    monkeypatch.setenv("OTK_EXTERNAL_PATH", "./external")
+    monkeypatch.setenv("OTK_UNDER_TEST", "1")
+
+    input_otk_path = tmp_path / "input.otk"
+    # keep in sync with "gen-images-ref"
+    with input_otk_path.open("w") as fp:
+        fp.write(textwrap.dedent(f"""
+        otk.define:
+          user:
+           modifications:
+            locale: nl_NL.UTF-8
+        otk.include: {tc.as_example_yaml()}
+        """))
+
+    with tc.ref_yaml_path.open() as fp:
+        ref_manifest = yaml.safe_load(fp)
+        normalize_rpm_refs(ref_manifest)
+
+    otk_json = tmp_path / "manifest-otk.json"
+    run(["compile",
+         "-o", os.fspath(otk_json),
+         os.fspath(input_otk_path),
          ])
     with otk_json.open() as fp:
         manifest = json.load(fp)