diff --git a/pkg/distro/fedora/images.go b/pkg/distro/fedora/images.go index 8b1be8b618..f91342114c 100644 --- a/pkg/distro/fedora/images.go +++ b/pkg/distro/fedora/images.go @@ -479,7 +479,7 @@ func imageInstallerImage(workload workload.Workload, img.Filename = t.Filename() - img.SquashfsCompression = "lz4" + img.RootfsCompression = "lz4" if common.VersionGreaterThanOrEqual(img.OSVersion, VERSION_ROOTFS_SQUASHFS) { img.RootfsType = manifest.SquashfsRootfs } @@ -680,7 +680,7 @@ func iotInstallerImage(workload workload.Workload, img.Filename = t.Filename() - img.SquashfsCompression = "lz4" + img.RootfsCompression = "lz4" if common.VersionGreaterThanOrEqual(img.OSVersion, VERSION_ROOTFS_SQUASHFS) { img.RootfsType = manifest.SquashfsRootfs } diff --git a/pkg/distro/rhel/images.go b/pkg/distro/rhel/images.go index e043250019..e6fb7bb9b8 100644 --- a/pkg/distro/rhel/images.go +++ b/pkg/distro/rhel/images.go @@ -495,7 +495,7 @@ func EdgeInstallerImage(workload workload.Workload, // kickstart though kickstart does support setting them img.Kickstart.Timezone, _ = customizations.GetTimezoneSettings() - img.SquashfsCompression = "xz" + img.RootfsCompression = "xz" if t.Arch().Distro().Releasever() == "10" { img.RootfsType = manifest.SquashfsRootfs } @@ -716,7 +716,7 @@ func ImageInstallerImage(workload workload.Workload, } img.AdditionalAnacondaModules = append(img.AdditionalAnacondaModules, anaconda.ModuleUsers) - img.SquashfsCompression = "xz" + img.RootfsCompression = "xz" if t.Arch().Distro().Releasever() == "10" { img.RootfsType = manifest.SquashfsRootfs } diff --git a/pkg/image/anaconda_container_installer.go b/pkg/image/anaconda_container_installer.go index 4d7b177d3f..1eef7c25b0 100644 --- a/pkg/image/anaconda_container_installer.go +++ b/pkg/image/anaconda_container_installer.go @@ -22,8 +22,8 @@ type AnacondaContainerInstaller struct { Platform platform.Platform ExtraBasePackages rpmmd.PackageSet - SquashfsCompression string - RootfsType manifest.RootfsType + RootfsCompression string + RootfsType manifest.RootfsType ISOLabel string Product string @@ -132,7 +132,8 @@ func (img *AnacondaContainerInstaller) InstantiateManifest(m *manifest.Manifest, isoTreePipeline.Release = img.Release isoTreePipeline.Kickstart = img.Kickstart - isoTreePipeline.SquashfsCompression = img.SquashfsCompression + isoTreePipeline.RootfsCompression = img.RootfsCompression + isoTreePipeline.RootfsType = img.RootfsType // For ostree installers, always put the kickstart file in the root of the ISO isoTreePipeline.PayloadPath = "/container" diff --git a/pkg/image/anaconda_live_installer.go b/pkg/image/anaconda_live_installer.go index 77738d98c2..b51d6d06c1 100644 --- a/pkg/image/anaconda_live_installer.go +++ b/pkg/image/anaconda_live_installer.go @@ -23,8 +23,8 @@ type AnacondaLiveInstaller struct { ExtraBasePackages rpmmd.PackageSet - SquashfsCompression string - RootfsType manifest.RootfsType + RootfsCompression string + RootfsType manifest.RootfsType ISOLabel string Product string @@ -107,7 +107,8 @@ func (img *AnacondaLiveInstaller) InstantiateManifest(m *manifest.Manifest, isoTreePipeline.KernelOpts = kernelOpts isoTreePipeline.ISOLinux = isoLinuxEnabled - isoTreePipeline.SquashfsCompression = img.SquashfsCompression + isoTreePipeline.RootfsCompression = img.RootfsCompression + isoTreePipeline.RootfsType = img.RootfsType isoPipeline := manifest.NewISO(buildPipeline, isoTreePipeline, img.ISOLabel) isoPipeline.SetFilename(img.Filename) diff --git a/pkg/image/anaconda_ostree_installer.go b/pkg/image/anaconda_ostree_installer.go index fc53cc25d8..a5587c2421 100644 --- a/pkg/image/anaconda_ostree_installer.go +++ b/pkg/image/anaconda_ostree_installer.go @@ -28,8 +28,8 @@ type AnacondaOSTreeInstaller struct { // Subscription options to include Subscription *subscription.ImageOptions - SquashfsCompression string - RootfsType manifest.RootfsType + RootfsCompression string + RootfsType manifest.RootfsType ISOLabel string Product string @@ -139,7 +139,8 @@ func (img *AnacondaOSTreeInstaller) InstantiateManifest(m *manifest.Manifest, isoTreePipeline.PartitionTable = efiBootPartitionTable(rng) isoTreePipeline.Release = img.Release isoTreePipeline.Kickstart = img.Kickstart - isoTreePipeline.SquashfsCompression = img.SquashfsCompression + isoTreePipeline.RootfsCompression = img.RootfsCompression + isoTreePipeline.RootfsType = img.RootfsType isoTreePipeline.PayloadPath = "/ostree/repo" diff --git a/pkg/image/anaconda_tar_installer.go b/pkg/image/anaconda_tar_installer.go index ad3d3b5c59..632b23d570 100644 --- a/pkg/image/anaconda_tar_installer.go +++ b/pkg/image/anaconda_tar_installer.go @@ -56,8 +56,8 @@ type AnacondaTarInstaller struct { ISORootKickstart bool Kickstart *kickstart.Options - SquashfsCompression string - RootfsType manifest.RootfsType + RootfsCompression string + RootfsType manifest.RootfsType ISOLabel string Product string @@ -195,7 +195,8 @@ func (img *AnacondaTarInstaller) InstantiateManifest(m *manifest.Manifest, isoTreePipeline.Kickstart.Path = img.Kickstart.Path } - isoTreePipeline.SquashfsCompression = img.SquashfsCompression + isoTreePipeline.RootfsCompression = img.RootfsCompression + isoTreePipeline.RootfsType = img.RootfsType isoTreePipeline.OSPipeline = osPipeline isoTreePipeline.KernelOpts = img.AdditionalKernelOpts diff --git a/pkg/manifest/anaconda_installer_iso_tree.go b/pkg/manifest/anaconda_installer_iso_tree.go index 9240e2d1a5..dafa2f2152 100644 --- a/pkg/manifest/anaconda_installer_iso_tree.go +++ b/pkg/manifest/anaconda_installer_iso_tree.go @@ -23,6 +23,7 @@ type RootfsType uint64 const ( // Rootfs type enum SquashfsExt4Rootfs RootfsType = iota // Create an EXT4 rootfs compressed by Squashfs SquashfsRootfs // Create a plain squashfs rootfs + ErofsRootfs // Create a plain erofs rootfs ) // An AnacondaInstallerISOTree represents a tree containing the anaconda installer, @@ -49,7 +50,8 @@ type AnacondaInstallerISOTree struct { isoLabel string - SquashfsCompression string + RootfsCompression string + RootfsType RootfsType OSPipeline *OS OSTreeCommitSource *ostree.SourceSpec @@ -135,8 +137,13 @@ func (p *AnacondaInstallerISOTree) getInline() []string { return inlineData } func (p *AnacondaInstallerISOTree) getBuildPackages(_ Distro) []string { - packages := []string{ - "squashfs-tools", + var packages []string + switch p.RootfsType { + case SquashfsExt4Rootfs, SquashfsRootfs: + packages = []string{"squashfs-tools"} + case ErofsRootfs: + packages = []string{"erofs-utils"} + default: } if p.OSTreeCommitSource != nil { @@ -154,6 +161,72 @@ func (p *AnacondaInstallerISOTree) getBuildPackages(_ Distro) []string { return packages } +// NewSquashfsStage returns an osbuild stage configured to build +// the squashfs root filesystem for the ISO. +func (p *AnacondaInstallerISOTree) NewSquashfsStage() *osbuild.Stage { + var squashfsOptions osbuild.SquashfsStageOptions + + if p.anacondaPipeline.Type == AnacondaInstallerTypePayload { + squashfsOptions = osbuild.SquashfsStageOptions{ + Filename: "images/install.img", + } + } else if p.anacondaPipeline.Type == AnacondaInstallerTypeLive { + squashfsOptions = osbuild.SquashfsStageOptions{ + Filename: "LiveOS/squashfs.img", + } + } + + if p.RootfsCompression != "" { + squashfsOptions.Compression.Method = p.RootfsCompression + } else { + // default to xz if not specified + squashfsOptions.Compression.Method = "xz" + } + + if squashfsOptions.Compression.Method == "xz" { + squashfsOptions.Compression.Options = &osbuild.FSCompressionOptions{ + BCJ: osbuild.BCJOption(p.anacondaPipeline.platform.GetArch().String()), + } + } + + // The iso's rootfs can either be an ext4 filesystem compressed with squashfs, or + // a squashfs of the plain directory tree + if p.RootfsType == SquashfsExt4Rootfs && p.rootfsPipeline != nil { + return osbuild.NewSquashfsStage(&squashfsOptions, p.rootfsPipeline.Name()) + } + return osbuild.NewSquashfsStage(&squashfsOptions, p.anacondaPipeline.Name()) +} + +// NewErofsStage returns an osbuild stage configured to build +// the erofs root filesystem for the ISO. +func (p *AnacondaInstallerISOTree) NewErofsStage() *osbuild.Stage { + var erofsOptions osbuild.ErofsStageOptions + + if p.anacondaPipeline.Type == AnacondaInstallerTypePayload { + erofsOptions = osbuild.ErofsStageOptions{ + Filename: "images/install.img", + } + } else if p.anacondaPipeline.Type == AnacondaInstallerTypeLive { + erofsOptions = osbuild.ErofsStageOptions{ + Filename: "LiveOS/squashfs.img", + } + } + + var compression osbuild.ErofsCompression + if p.RootfsCompression != "" { + compression.Method = p.RootfsCompression + } else { + // default to zstd if not specified + compression.Method = "zstd" + } + compression.Level = common.ToPtr(8) + erofsOptions.Compression = &compression + erofsOptions.ExtendedOptions = []string{"all-fragments", "dedupe"} + erofsOptions.ClusterSize = common.ToPtr(131072) + + return osbuild.NewErofsStage(&erofsOptions, p.anacondaPipeline.Name()) +} + func (p *AnacondaInstallerISOTree) serializeStart(_ []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec, _ []rpmmd.RepoConfig) { if p.ostreeCommitSpec != nil || p.containerSpec != nil { panic("double call to serializeStart()") @@ -261,40 +334,14 @@ func (p *AnacondaInstallerISOTree) serialize() osbuild.Pipeline { copyStage := osbuild.NewCopyStageSimple(copyStageOptions, copyStageInputs) pipeline.AddStage(copyStage) - var squashfsOptions osbuild.SquashfsStageOptions - - if p.anacondaPipeline.Type == AnacondaInstallerTypePayload { - squashfsOptions = osbuild.SquashfsStageOptions{ - Filename: "images/install.img", - } - } else if p.anacondaPipeline.Type == AnacondaInstallerTypeLive { - squashfsOptions = osbuild.SquashfsStageOptions{ - Filename: "LiveOS/squashfs.img", - } - } - - if p.SquashfsCompression != "" { - squashfsOptions.Compression.Method = p.SquashfsCompression - } else { - // default to xz if not specified - squashfsOptions.Compression.Method = "xz" - } - - if squashfsOptions.Compression.Method == "xz" { - squashfsOptions.Compression.Options = &osbuild.FSCompressionOptions{ - BCJ: osbuild.BCJOption(p.anacondaPipeline.platform.GetArch().String()), - } - } - - // The iso's rootfs can either be an ext4 filesystem compressed with squashfs, or - // a squashfs of the plain directory tree - var squashfsStage *osbuild.Stage - if p.rootfsPipeline != nil { - squashfsStage = osbuild.NewSquashfsStage(&squashfsOptions, p.rootfsPipeline.Name()) - } else { - squashfsStage = osbuild.NewSquashfsStage(&squashfsOptions, p.anacondaPipeline.Name()) + // Add the selected roofs stage + switch p.RootfsType { + case SquashfsExt4Rootfs, SquashfsRootfs: + pipeline.AddStage(p.NewSquashfsStage()) + case ErofsRootfs: + pipeline.AddStage(p.NewErofsStage()) + default: } - pipeline.AddStage(squashfsStage) if p.ISOLinux { isoLinuxOptions := &osbuild.ISOLinuxStageOptions{ diff --git a/pkg/manifest/anaconda_installer_iso_tree_test.go b/pkg/manifest/anaconda_installer_iso_tree_test.go index 511ae63bff..26ce76828d 100644 --- a/pkg/manifest/anaconda_installer_iso_tree_test.go +++ b/pkg/manifest/anaconda_installer_iso_tree_test.go @@ -4,6 +4,8 @@ import ( "crypto/sha256" "fmt" "math/rand" + "slices" + "strings" "testing" "github.com/osbuild/images/internal/common" @@ -71,6 +73,16 @@ func newTestAnacondaISOTree() *AnacondaInstallerISOTree { return pipeline } +// Helper to return a comma separated string of the stage names +// used to help debug failures +func dumpStages(stages []*osbuild.Stage) string { + var stageNames []string + for _, stage := range stages { + stageNames = append(stageNames, stage.Type) + } + return strings.Join(stageNames, ", ") +} + func checkISOTreeStages(stages []*osbuild.Stage, expected, exclude []string) error { commonStages := []string{ "org.osbuild.mkdir", @@ -83,6 +95,13 @@ func checkISOTreeStages(stages []*osbuild.Stage, expected, exclude []string) err "org.osbuild.discinfo", } + // Remove excluded stages from common + for _, exlStage := range exclude { + if idx := slices.Index(commonStages, exlStage); idx > -1 { + commonStages = slices.Delete(commonStages, idx, idx+1) + } + } + for _, expStage := range append(commonStages, expected...) { if findStage(expStage, stages) == nil { return fmt.Errorf("did not find expected stage: %s", expStage) @@ -404,6 +423,31 @@ func TestAnacondaISOTreeSerializeWithOS(t *testing.T) { pipeline.serializeStart(nil, nil, nil, nil) assert.Panics(t, func() { pipeline.serialize() }) }) + + t.Run("plain+squashfs-rootfs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.OSPipeline = osPayload + pipeline.RootfsType = SquashfsRootfs + pipeline.serializeStart(nil, nil, nil, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, + append(variantStages, []string{"org.osbuild.kickstart", "org.osbuild.isolinux"}...)), + dumpStages(sp.Stages)) + }) + + t.Run("plain+erofs-rootfs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.OSPipeline = osPayload + pipeline.RootfsType = ErofsRootfs + pipeline.serializeStart(nil, nil, nil, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, + append(payloadStages, "org.osbuild.erofs"), + append(variantStages, []string{"org.osbuild.kickstart", "org.osbuild.isolinux", "org.osbuild.squashfs"}...)), + dumpStages(sp.Stages)) + }) } func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { @@ -524,6 +568,31 @@ func TestAnacondaISOTreeSerializeWithOSTree(t *testing.T) { pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) assert.Panics(t, func() { pipeline.serialize() }) }) + + t.Run("plain+squashfs-rootfs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.RootfsType = SquashfsRootfs + pipeline.Kickstart = &kickstart.Options{Path: testKsPath, OSTree: &kickstart.OSTree{}} + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, + append(variantStages, "org.osbuild.isolinux")), + dumpStages(sp.Stages)) + }) + + t.Run("plain+erofs-erofs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.RootfsType = ErofsRootfs + pipeline.Kickstart = &kickstart.Options{Path: testKsPath, OSTree: &kickstart.OSTree{}} + pipeline.serializeStart(nil, nil, []ostree.CommitSpec{ostreeCommit}, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, + append(payloadStages, "org.osbuild.erofs"), + append(variantStages, []string{"org.osbuild.isolinux", "org.osbuild.squashfs"}...)), + dumpStages(sp.Stages)) + }) } func makeFakeContainerPayload() container.Spec { @@ -627,6 +696,31 @@ func TestAnacondaISOTreeSerializeWithContainer(t *testing.T) { assert.NotNil(t, skopeoStage) assert.Equal(t, skopeoStage.Options.(*osbuild.SkopeoStageOptions).RemoveSignatures, common.ToPtr(true)) }) + + t.Run("plain+squashfs-rootfs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.RootfsType = SquashfsRootfs + pipeline.Kickstart = &kickstart.Options{Path: testKsPath} + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, payloadStages, + append(variantStages, "org.osbuild.isolinux")), + dumpStages(sp.Stages)) + }) + + t.Run("plain+erofs-rootfs", func(t *testing.T) { + pipeline := newTestAnacondaISOTree() + pipeline.RootfsType = ErofsRootfs + pipeline.Kickstart = &kickstart.Options{Path: testKsPath} + pipeline.serializeStart(nil, []container.Spec{containerPayload}, nil, nil) + sp := pipeline.serialize() + pipeline.serializeEnd() + assert.NoError(t, checkISOTreeStages(sp.Stages, + append(payloadStages, "org.osbuild.erofs"), + append(variantStages, []string{"org.osbuild.isolinux", "org.osbuild.squashfs"}...)), + dumpStages(sp.Stages)) + }) } func TestMakeKickstartSudoersPostEmpty(t *testing.T) { diff --git a/pkg/osbuild/erofs_stage.go b/pkg/osbuild/erofs_stage.go new file mode 100644 index 0000000000..fafd19bc31 --- /dev/null +++ b/pkg/osbuild/erofs_stage.go @@ -0,0 +1,24 @@ +package osbuild + +type ErofsCompression struct { + Method string `json:"method"` + Level *int `json:"level,omitempty"` +} + +type ErofsStageOptions struct { + Filename string `json:"filename"` + + Compression *ErofsCompression `json:"compression,omitempty"` + ExtendedOptions []string `json:"options,omitempty"` + ClusterSize *int `json:"cluster-size,omitempty"` +} + +func (ErofsStageOptions) isStageOptions() {} + +func NewErofsStage(options *ErofsStageOptions, inputPipeline string) *Stage { + return &Stage{ + Type: "org.osbuild.erofs", + Options: options, + Inputs: NewPipelineTreeInputs("tree", inputPipeline), + } +} diff --git a/pkg/osbuild/erofs_stage_test.go b/pkg/osbuild/erofs_stage_test.go new file mode 100644 index 0000000000..b86356e3eb --- /dev/null +++ b/pkg/osbuild/erofs_stage_test.go @@ -0,0 +1,83 @@ +package osbuild_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/images/internal/common" + "github.com/osbuild/images/pkg/osbuild" +) + +func TestErofStageJsonMinimal(t *testing.T) { + expectedJson := `{ + "type": "org.osbuild.erofs", + "inputs": { + "tree": { + "type": "org.osbuild.tree", + "origin": "org.osbuild.pipeline", + "references": [ + "name:input-pipeline" + ] + } + }, + "options": { + "filename": "foo.ero" + } +}` + + opts := &osbuild.ErofsStageOptions{ + Filename: "foo.ero", + } + stage := osbuild.NewErofsStage(opts, "input-pipeline") + require.NotNil(t, stage) + + json, err := json.MarshalIndent(stage, "", " ") + require.Nil(t, err) + assert.Equal(t, string(json), expectedJson) +} + +func TestErofStageJsonFull(t *testing.T) { + expectedJson := `{ + "type": "org.osbuild.erofs", + "inputs": { + "tree": { + "type": "org.osbuild.tree", + "origin": "org.osbuild.pipeline", + "references": [ + "name:input-pipeline" + ] + } + }, + "options": { + "filename": "foo.ero", + "compression": { + "method": "lz4hc", + "level": 9 + }, + "options": [ + "all-fragments", + "dedupe" + ], + "cluster-size": 131072 + } +}` + + opts := &osbuild.ErofsStageOptions{ + Filename: "foo.ero", + Compression: &osbuild.ErofsCompression{ + Method: "lz4hc", + Level: common.ToPtr(9), + }, + ExtendedOptions: []string{"all-fragments", "dedupe"}, + ClusterSize: common.ToPtr(131072), + } + stage := osbuild.NewErofsStage(opts, "input-pipeline") + require.NotNil(t, stage) + + json, err := json.MarshalIndent(stage, "", " ") + require.Nil(t, err) + assert.Equal(t, string(json), expectedJson) +}