smoke: add nydusd hot upgrade test case

smoke/Makefile

@@ -13,6 +13,7 @@ build:
 # NYDUS_BUILDER=/path/to/latest/nydus-image \
 # NYDUS_NYDUSD=/path/to/latest/nydusd \
 # NYDUS_NYDUSIFY=/path/to/latest/nydusify \
+# SKIP_CASES=compressor=lz4_block,fs_version=5 \
 # make test
 test: build
 	golangci-lint run

smoke/tests/api_test.go

@@ -5,13 +5,11 @@
 package tests
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/converter"
@@ -33,7 +31,7 @@ func (a *APIV1TestSuite) TestDaemonStatus(t *testing.T) {
 
 	rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs"))
 
-	rafs := a.rootFsToRafs(t, ctx, rootFs)
+	rafs := a.buildLayer(t, ctx, rootFs)
 
 	nydusd, err := tool.NewNydusd(tool.NydusdConfig{
 		NydusdPath:      ctx.Binary.Nydusd,
@@ -60,17 +58,8 @@ func (a *APIV1TestSuite) TestDaemonStatus(t *testing.T) {
 		}
 	}()
 
-	// The implementation of runNydusd() has checked stats, however,
-	// it's clear of semantic to check stats again.
-	newCtx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	select {
-	case <-tool.CheckReady(newCtx, nydusd.APISockPath):
-		return
-	case <-time.After(50 * time.Millisecond):
-		require.Fail(t, "nydusd status is not RUNNING")
-	}
+	err = nydusd.WaitStatus("RUNNING")
+	require.NoError(t, err)
 }
 
 func (a *APIV1TestSuite) TestMetrics(t *testing.T) {
@@ -82,7 +71,7 @@ func (a *APIV1TestSuite) TestMetrics(t *testing.T) {
 
 	rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs"))
 
-	rafs := a.rootFsToRafs(t, ctx, rootFs)
+	rafs := a.buildLayer(t, ctx, rootFs)
 
 	nydusd, err := tool.NewNydusd(tool.NydusdConfig{
 		NydusdPath:      ctx.Binary.Nydusd,
@@ -164,7 +153,7 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) {
 		filepath.Join(ctx.Env.WorkDir, "root-fs"),
 		texture.LargerFileMaker("large-blob.bin", 5))
 
-	rafs := a.rootFsToRafs(t, ctx, rootFs)
+	rafs := a.buildLayer(t, ctx, rootFs)
 
 	config := tool.NydusdConfig{
 		NydusdPath:  ctx.Binary.Nydusd,
@@ -195,7 +184,6 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) {
 	config.RafsMode = ctx.Runtime.RafsMode
 	err = nydusd.MountByAPI(config)
 	require.NoError(t, err)
-	time.Sleep(time.Millisecond * 15)
 
 	bcm, err := nydusd.GetBlobCacheMetrics("")
 	require.NoError(t, err)
@@ -205,7 +193,7 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func (a *APIV1TestSuite) rootFsToRafs(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string {
+func (a *APIV1TestSuite) buildLayer(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string {
 	digest := rootFs.Pack(t,
 		converter.PackOption{
 			BuilderPath: ctx.Binary.Builder,

smoke/tests/commit_test.go

@@ -36,7 +36,7 @@ func (c *CommitTestSuite) TestCommitContainer() test.Generator {
 		ctx.Build.FSVersion = scenario.GetString(paramFSVersion)
 
 		image, committedImage := c.prepareImage(c.t, ctx, scenario.GetString(paramImage))
-		return scenario.Str(), func(t *testing.T) {
+		return scenario.Str(), func(_ *testing.T) {
 			c.TestCommitAndCheck(*ctx, image, committedImage)
 		}
 	}

smoke/tests/hot_upgrade_test.go

@@ -0,0 +1,156 @@
+// Copyright 2024 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package tests
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/containerd/nydus-snapshotter/pkg/converter"
+	"github.com/containerd/nydus-snapshotter/pkg/supervisor"
+	"github.com/dragonflyoss/nydus/smoke/tests/texture"
+	"github.com/dragonflyoss/nydus/smoke/tests/tool"
+	"github.com/dragonflyoss/nydus/smoke/tests/tool/test"
+	"github.com/stretchr/testify/require"
+)
+
+type Snapshotter struct {
+}
+
+type HotUpgradeTestSuite struct {
+	t *testing.T
+}
+
+func (c *HotUpgradeTestSuite) buildLayer(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string {
+	digest := rootFs.Pack(t,
+		converter.PackOption{
+			BuilderPath: ctx.Binary.Builder,
+			Compressor:  "lz4_block",
+			FsVersion:   "5",
+		},
+		ctx.Env.BlobDir)
+	_, bootstrap := tool.MergeLayers(t, *ctx,
+		converter.MergeOption{
+			BuilderPath: ctx.Binary.Builder,
+		},
+		[]converter.Layer{
+			{Digest: digest},
+		})
+	return bootstrap
+}
+
+func (c *HotUpgradeTestSuite) newNydusd(t *testing.T, ctx *tool.Context, bootstrap, name string, upgrade bool) *tool.Nydusd {
+	config := tool.NydusdConfig{
+		NydusdPath:         ctx.Binary.Nydusd,
+		MountPath:          ctx.Env.MountDir,
+		APISockPath:        filepath.Join(ctx.Env.WorkDir, fmt.Sprintf("nydusd-api-%s.sock", name)),
+		ConfigPath:         filepath.Join(ctx.Env.WorkDir, fmt.Sprintf("nydusd-config.fusedev-%s.json", name)),
+		SupervisorSockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-supervisor.sock"),
+	}
+	if upgrade {
+		config.Upgrade = true
+	}
+
+	nydusd, err := tool.NewNydusd(config)
+	require.NoError(t, err)
+
+	_, err = nydusd.Run()
+	require.NoError(t, err)
+
+	if upgrade {
+		err = nydusd.WaitStatus("INIT")
+	} else {
+		err = nydusd.WaitStatus("RUNNING")
+	}
+	require.NoError(t, err)
+
+	config.BootstrapPath = bootstrap
+	config.MountPath = "/"
+	config.BackendType = "localfs"
+	config.BackendConfig = fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir)
+	config.EnablePrefetch = true
+	config.PrefetchFiles = []string{"/"}
+	config.BlobCacheDir = ctx.Env.CacheDir
+	config.CacheType = ctx.Runtime.CacheType
+	config.CacheCompressed = ctx.Runtime.CacheCompressed
+	config.RafsMode = ctx.Runtime.RafsMode
+
+	err = nydusd.MountByAPI(config)
+	require.NoError(t, err)
+
+	return nydusd
+}
+
+func (c *HotUpgradeTestSuite) TestHotUpgrade(t *testing.T) {
+	ctx := tool.DefaultContext(t)
+	ctx.PrepareWorkDir(t)
+	defer ctx.Destroy(t)
+
+	// Build nydus layer
+	layer := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root"))
+	bootstrap := c.buildLayer(t, ctx, layer)
+
+	// Start snapshotter simulator
+	ss, err := supervisor.NewSupervisorSet(filepath.Join(ctx.Env.WorkDir))
+	require.NoError(t, err)
+	supervisor := ss.NewSupervisor("nydusd-supervisor")
+	defer ss.DestroySupervisor("nydusd-supervisor")
+
+	// Start old nydusd to mount rootfs
+	oldNydusd := c.newNydusd(t, ctx, bootstrap, "old", false)
+	defer oldNydusd.Umount()
+
+	// Old nydusd's state should be RUNNING
+	err = oldNydusd.WaitStatus("RUNNING")
+	require.NoError(t, err)
+
+	// Verify filesytem on new nydusd
+	oldNydusd.Verify(t, layer.FileTree)
+
+	// Snapshotter receive fuse fd from old nydusd
+	err = supervisor.FetchDaemonStates(oldNydusd.SendFd)
+	require.NoError(t, err)
+
+	// Start new nydusd in upgrade mode (don't mount)
+	newNydusd := c.newNydusd(t, ctx, bootstrap, "new", true)
+	defer newNydusd.Umount()
+
+	// New nydusd's state should be INIT
+	err = newNydusd.WaitStatus("INIT")
+	require.NoError(t, err)
+
+	// Tells old nydusd to exit
+	err = oldNydusd.Exit()
+	require.NoError(t, err)
+
+	// Send fuse fd to new nydusd
+	err = supervisor.SendStatesTimeout(time.Second * 5)
+	require.NoError(t, err)
+	err = newNydusd.Takeover()
+	require.NoError(t, err)
+
+	// New nydusd's state should be RUNNING | READY
+	// Only have RUNNING state for older nydusd version (v1.x)
+	err = newNydusd.WaitStatus("RUNNING", "READY")
+	require.NoError(t, err)
+
+	// Snapshotter receive fuse fd from new nydusd
+	err = supervisor.FetchDaemonStates(newNydusd.SendFd)
+	require.NoError(t, err)
+
+	// Start new nydusd to serve mountpoint
+	// It's unnecessary for older nydusd version (v1.x)
+	err = newNydusd.StartByAPI()
+	require.NoError(t, err)
+
+	// Verify filesytem on new nydusd
+	newNydusd.Verify(t, layer.FileTree)
+}
+
+func TestHotUpgrade(t *testing.T) {
+	test.Run(t, &HotUpgradeTestSuite{t: t})
+}

smoke/tests/main_test.go

@@ -22,8 +22,10 @@ func TestMain(m *testing.M) {
 		registryPort = "5077"
 		os.Setenv("REGISTRY_PORT", registryPort)
 	}
-	reg := tool.NewRegistry()
+	if os.Getenv("DISABLE_REGISTRY") == "" {
+		reg := tool.NewRegistry()
+		defer reg.Destroy()
+	}
 	code := m.Run()
-	reg.Destroy()
 	os.Exit(code)
 }

smoke/tests/native_layer_test.go

@@ -45,7 +45,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator {
 		Dimension(paramEncrypt, []interface{}{false, true}).
 		Dimension(paramAmplifyIO, []interface{}{uint64(0x100000)}).
 		Skip(func(param *tool.DescartesItem) bool {
-
 			// rafs v6 not support cached mode nor dummy cache
 			if param.GetString(paramFSVersion) == "6" {
 				return param.GetString(paramRafsMode) == "cached" || param.GetString(paramCacheType) == ""
@@ -70,17 +69,16 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator {
 		}
 		scenario := scenarios.Next()
 
-		ctx := tool.DefaultContext(n.t)
-		ctx.Build.Compressor = scenario.GetString(paramCompressor)
-		ctx.Build.FSVersion = scenario.GetString(paramFSVersion)
-		ctx.Build.ChunkSize = scenario.GetString(paramChunkSize)
-		ctx.Runtime.CacheType = scenario.GetString(paramCacheType)
-		ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed)
-		ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode)
-		ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch)
-		ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO)
-
 		return scenario.Str(), func(t *testing.T) {
+			ctx := tool.DefaultContext(n.t)
+			ctx.Build.Compressor = scenario.GetString(paramCompressor)
+			ctx.Build.FSVersion = scenario.GetString(paramFSVersion)
+			ctx.Build.ChunkSize = scenario.GetString(paramChunkSize)
+			ctx.Runtime.CacheType = scenario.GetString(paramCacheType)
+			ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed)
+			ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode)
+			ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch)
+			ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO)
 			n.testMakeLayers(*ctx, t)
 		}
 	}
@@ -105,7 +103,6 @@ func (n *NativeLayerTestSuite) TestAmplifyIO() test.Generator {
 		/* Amplify io - target param */
 		Dimension(paramAmplifyIO, []interface{}{uint64(0x0), uint64(0x100000), uint64(0x10000000)}).
 		Skip(func(param *tool.DescartesItem) bool {
-
 			// Rafs v6 not support cached mode nor dummy cache
 			if param.GetString(paramFSVersion) == "6" {
 				return param.GetString(paramRafsMode) == "cached" || param.GetString(paramCacheType) == ""
@@ -130,17 +127,16 @@ func (n *NativeLayerTestSuite) TestAmplifyIO() test.Generator {
 		}
 		scenario := scenarios.Next()
 
-		ctx := tool.DefaultContext(n.t)
-		ctx.Build.Compressor = scenario.GetString(paramCompressor)
-		ctx.Build.FSVersion = scenario.GetString(paramFSVersion)
-		ctx.Build.ChunkSize = scenario.GetString(paramChunkSize)
-		ctx.Runtime.CacheType = scenario.GetString(paramCacheType)
-		ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed)
-		ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode)
-		ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch)
-		ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO)
-
 		return scenario.Str(), func(t *testing.T) {
+			ctx := tool.DefaultContext(n.t)
+			ctx.Build.Compressor = scenario.GetString(paramCompressor)
+			ctx.Build.FSVersion = scenario.GetString(paramFSVersion)
+			ctx.Build.ChunkSize = scenario.GetString(paramChunkSize)
+			ctx.Runtime.CacheType = scenario.GetString(paramCacheType)
+			ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed)
+			ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode)
+			ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch)
+			ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO)
 			n.testMakeLayers(*ctx, t)
 		}
 	}

smoke/tests/overlay_fs_test.go

@@ -51,6 +51,7 @@ func (ts *OverlayFsTestSuite) prepareTestEnv(t *testing.T) *tool.Context {
 	// Verify lower layer mounted by nydusd
 	ctx.Env.BootstrapPath = lowerBootstrap
 	tool.Verify(t, *ctx, lowerLayer.FileTree)
+
 	return ctx
 }
 

smoke/tests/tool/iterator.go

@@ -2,10 +2,25 @@ package tool
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 )
 
+func isIgnoredByEnv(param *DescartesItem) bool {
+	if skipCases := os.Getenv("SKIP_CASES"); skipCases != "" {
+		kvs := strings.Split(skipCases, ",")
+		for _, kv := range kvs {
+			k := strings.Split(kv, "=")[0]
+			v := strings.Split(kv, "=")[1]
+			if param.GetString(k) == v {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 type DescartesItem struct {
 	vals map[string]interface{}
 }
@@ -141,7 +156,7 @@ func (c *DescartesIterator) calNext() {
 		for name, idx := range c.cursorMap {
 			item.vals[name] = c.valLists[idx][cursors[idx]]
 		}
-		if c.skip == nil || !c.skip(item) {
+		if !isIgnoredByEnv(item) && (c.skip == nil || !c.skip(item)) {
 			c.haveNext(cursors, item)
 			return
 		}

smoke/tests/tool/layer.go

@@ -232,6 +232,9 @@ func (l *Layer) recordFileTree(t *testing.T) {
 	l.FileTree = map[string]*File{}
 	filepath.Walk(l.workDir, func(path string, _ os.FileInfo, _ error) error {
 		targetPath := l.TargetPath(t, path)
+		if targetPath == "." || targetPath == ".." {
+			return nil
+		}
 		l.FileTree[targetPath] = NewFile(t, path, targetPath)
 		return nil
 	})