From df846407dc77c33af467ff69fe8ca5145a5d5535 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sat, 2 Nov 2024 01:18:30 +0200 Subject: [PATCH] Test qcow2 images with different allocation Add a helper for creating an image from list of extents. With this we can create a raw or qcow2 image with allocation described by the extents. Add 4 tests, testing different allocation patterns: - TestExtentsSome: some allocated clusters and some holes - TestExtentsPartial: writing partial cluster allocates entire cluster - TestExtentsMerge: consecutive extents of same type are merged - TestExtentsZero: different extents types that read as zeros For each test we verify qcow2 and qcow2 compressed (zlib) images. Signed-off-by: Nir Soffer --- qcow2reader_test.go | 358 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) diff --git a/qcow2reader_test.go b/qcow2reader_test.go index 89213fb..71e8a7b 100644 --- a/qcow2reader_test.go +++ b/qcow2reader_test.go @@ -7,15 +7,18 @@ import ( "math/rand" "os" "path/filepath" + "slices" "testing" "github.com/lima-vm/go-qcow2reader" "github.com/lima-vm/go-qcow2reader/convert" "github.com/lima-vm/go-qcow2reader/image" "github.com/lima-vm/go-qcow2reader/test/qemuimg" + "github.com/lima-vm/go-qcow2reader/test/qemuio" ) const ( + KiB = int64(1) << 10 MiB = int64(1) << 20 GiB = int64(1) << 30 ) @@ -159,6 +162,361 @@ func BenchmarkExtentsUnallocated(b *testing.B) { } } +func TestExtentsSome(t *testing.T) { + clusterSize := 64 * KiB + extents := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + { + Start: 1 * clusterSize, + Length: 1 * clusterSize, + Zero: true, + }, + { + Start: 2 * clusterSize, + Length: 2 * clusterSize, + Allocated: true, + }, + { + Start: 4 * clusterSize, + Length: 96 * clusterSize, + Zero: true, + }, + { + Start: 100 * clusterSize, + Length: 8 * clusterSize, + Allocated: true, + }, + { + Start: 108 * clusterSize, + Length: 892 * clusterSize, + Zero: true, + }, + { + Start: 1000 * clusterSize, + Length: 16 * clusterSize, + Allocated: true, + }, + { + Start: 1016 * clusterSize, + Length: 8984 * clusterSize, + Zero: true, + }, + } + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(extents, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(extents), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsPartial(t *testing.T) { + clusterSize := 64 * KiB + + // Writing part of of a cluster allocates entire cluster in the qcow2 image. + extents := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 1, + Allocated: true, + }, + { + Start: 1 * clusterSize, + Length: 98 * clusterSize, + Zero: true, + }, + { + Start: 100*clusterSize - 1, + Length: 1, + Allocated: true, + }, + } + + // Listing extents works in cluster granularity. + full := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + { + Start: 1 * clusterSize, + Length: 98 * clusterSize, + Zero: true, + }, + { + Start: 99 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(full, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(full), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsMerge(t *testing.T) { + clusterSize := 64 * KiB + + // Create image with consecutive extents of same type. + extents := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + { + Start: 1 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + { + Start: 2 * clusterSize, + Length: 98 * clusterSize, + Zero: true, + }, + { + Start: 100 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + { + Start: 101 * clusterSize, + Length: 1 * clusterSize, + Allocated: true, + }, + } + + // Extents with same type are merged. + merged := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 2 * clusterSize, + Allocated: true, + }, + { + Start: 2 * clusterSize, + Length: 98 * clusterSize, + Zero: true, + }, + { + Start: 100 * clusterSize, + Length: 2 * clusterSize, + Allocated: true, + }, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(merged, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(compressed(merged), actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func TestExtentsZero(t *testing.T) { + clusterSize := 64 * KiB + + // Create image with different clusters that read as zeros. + extents := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 1000 * clusterSize, + Allocated: true, + Zero: true, + }, + { + Start: 1000 * clusterSize, + Length: 1000 * clusterSize, + Zero: true, + }, + } + + qcow2 := filepath.Join(t.TempDir(), "image") + if err := createTestImageWithExtents(qcow2, qemuimg.FormatQcow2, extents); err != nil { + t.Fatal(err) + } + t.Run("qcow2", func(t *testing.T) { + actual, err := listExtents(qcow2) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(extents, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) + t.Run("qcow2 zlib", func(t *testing.T) { + qcow2Zlib := qcow2 + ".zlib" + if err := qemuimg.Convert(qcow2, qcow2Zlib, qemuimg.FormatQcow2, qemuimg.CompressionZlib); err != nil { + t.Fatal(err) + } + actual, err := listExtents(qcow2Zlib) + if err != nil { + t.Fatal(err) + } + // When converting to qcow2 images all clusters that read as zeros are + // converted to unallocated clusters. + converted := []image.Extent{ + { + Start: 0 * clusterSize, + Length: 2000 * clusterSize, + Zero: true, + }, + } + if !slices.Equal(converted, actual) { + t.Fatalf("expected %v, got %v", extents, actual) + } + }) +} + +func compressed(extents []image.Extent) []image.Extent { + var res []image.Extent + for _, extent := range extents { + if extent.Allocated { + extent.Compressed = true + } + res = append(res, extent) + } + return res +} + +func listExtents(path string) ([]image.Extent, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + img, err := qcow2reader.Open(f) + if err != nil { + return nil, err + } + defer img.Close() + + var extents []image.Extent + var start int64 + + end := img.Size() + for start < end { + extent, err := img.Next(start, end-start) + if err != nil { + return nil, err + } + if extent.Start != start { + return nil, fmt.Errorf("invalid extent start: %+v", extent) + } + if extent.Length <= 0 { + return nil, fmt.Errorf("invalid extent length: %+v", extent) + } + extents = append(extents, extent) + start += extent.Length + } + return extents, nil +} + +// createTestImageWithExtents creates a n image with the allocation described +// by extents. +func createTestImageWithExtents(path string, format qemuimg.Format, extents []image.Extent) error { + lastExtent := extents[len(extents)-1] + size := lastExtent.Start + lastExtent.Length + if err := qemuimg.Create(path, format, size, "", ""); err != nil { + return err + } + for _, extent := range extents { + if !extent.Allocated { + continue + } + start := extent.Start + length := extent.Length + for length > 0 { + // qemu-io requires length < 2g. + n := length + if n >= 2*GiB { + n = 2*GiB - 64*KiB + } + if extent.Zero { + if err := qemuio.Zero(path, format, start, n); err != nil { + return err + } + } else { + if err := qemuio.Write(path, format, start, n, 0x55); err != nil { + return err + } + } + start += n + length -= n + } + } + return nil +} + // Benchmark completely empty sparse image (0% utilization). This is the best // case when we don't have to read any cluster from storage. func Benchmark0p(b *testing.B) {