From 6c9ed19ecbc280cc6dd99c7080c33fc823e8d07e Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 18 Jul 2024 10:43:29 +0000 Subject: [PATCH 1/2] smoke: refactor nydusd methods for testing Renaming and adding some methods for nydusd struct, for easily controlling nydusd process. And support SKIP_CASES env to allow skipping some cases. Signed-off-by: Yan Song --- smoke/Makefile | 1 + smoke/tests/api_test.go | 24 +- smoke/tests/commit_test.go | 2 +- smoke/tests/main_test.go | 6 +- smoke/tests/native_layer_test.go | 40 ++- smoke/tests/overlay_fs_test.go | 1 + smoke/tests/tool/iterator.go | 17 +- smoke/tests/tool/layer.go | 3 + smoke/tests/tool/nydusd.go | 452 ++++++++++++++++--------------- smoke/tests/tool/verify.go | 71 ----- 10 files changed, 277 insertions(+), 340 deletions(-) delete mode 100644 smoke/tests/tool/verify.go diff --git a/smoke/Makefile b/smoke/Makefile index 05562e6f769..4703117b6cb 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -13,6 +13,7 @@ build: # NYDUS_BUILDER=/path/to/latest/nydus-image \ # NYDUS_NYDUSD=/path/to/latest/nydusd \ # NYDUS_NYDUSIFY=/path/to/latest/nydusify \ +# SKIP_CASES=compressor=lz4_block,fs_version=5 \ # make test test: build golangci-lint run diff --git a/smoke/tests/api_test.go b/smoke/tests/api_test.go index 33cea47e81a..bc05dc094e0 100644 --- a/smoke/tests/api_test.go +++ b/smoke/tests/api_test.go @@ -5,13 +5,11 @@ package tests import ( - "context" "fmt" "io" "os" "path/filepath" "testing" - "time" "github.com/containerd/log" "github.com/containerd/nydus-snapshotter/pkg/converter" @@ -33,7 +31,7 @@ func (a *APIV1TestSuite) TestDaemonStatus(t *testing.T) { rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) - rafs := a.rootFsToRafs(t, ctx, rootFs) + rafs := a.buildLayer(t, ctx, rootFs) nydusd, err := tool.NewNydusd(tool.NydusdConfig{ NydusdPath: ctx.Binary.Nydusd, @@ -60,17 +58,8 @@ func (a *APIV1TestSuite) TestDaemonStatus(t *testing.T) { } }() - // The implementation of runNydusd() has checked stats, however, - // it's clear of semantic to check stats again. - newCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - - select { - case <-tool.CheckReady(newCtx, nydusd.APISockPath): - return - case <-time.After(50 * time.Millisecond): - require.Fail(t, "nydusd status is not RUNNING") - } + err = nydusd.WaitStatus("RUNNING") + require.NoError(t, err) } func (a *APIV1TestSuite) TestMetrics(t *testing.T) { @@ -82,7 +71,7 @@ func (a *APIV1TestSuite) TestMetrics(t *testing.T) { rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) - rafs := a.rootFsToRafs(t, ctx, rootFs) + rafs := a.buildLayer(t, ctx, rootFs) nydusd, err := tool.NewNydusd(tool.NydusdConfig{ NydusdPath: ctx.Binary.Nydusd, @@ -164,7 +153,7 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) { filepath.Join(ctx.Env.WorkDir, "root-fs"), texture.LargerFileMaker("large-blob.bin", 5)) - rafs := a.rootFsToRafs(t, ctx, rootFs) + rafs := a.buildLayer(t, ctx, rootFs) config := tool.NydusdConfig{ NydusdPath: ctx.Binary.Nydusd, @@ -195,7 +184,6 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) { config.RafsMode = ctx.Runtime.RafsMode err = nydusd.MountByAPI(config) require.NoError(t, err) - time.Sleep(time.Millisecond * 15) bcm, err := nydusd.GetBlobCacheMetrics("") require.NoError(t, err) @@ -205,7 +193,7 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) { require.NoError(t, err) } -func (a *APIV1TestSuite) rootFsToRafs(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string { +func (a *APIV1TestSuite) buildLayer(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string { digest := rootFs.Pack(t, converter.PackOption{ BuilderPath: ctx.Binary.Builder, diff --git a/smoke/tests/commit_test.go b/smoke/tests/commit_test.go index 2c1a0d1e0b6..9d7579808c0 100644 --- a/smoke/tests/commit_test.go +++ b/smoke/tests/commit_test.go @@ -36,7 +36,7 @@ func (c *CommitTestSuite) TestCommitContainer() test.Generator { ctx.Build.FSVersion = scenario.GetString(paramFSVersion) image, committedImage := c.prepareImage(c.t, ctx, scenario.GetString(paramImage)) - return scenario.Str(), func(t *testing.T) { + return scenario.Str(), func(_ *testing.T) { c.TestCommitAndCheck(*ctx, image, committedImage) } } diff --git a/smoke/tests/main_test.go b/smoke/tests/main_test.go index e93598802a0..fa2a8b0be47 100644 --- a/smoke/tests/main_test.go +++ b/smoke/tests/main_test.go @@ -22,8 +22,10 @@ func TestMain(m *testing.M) { registryPort = "5077" os.Setenv("REGISTRY_PORT", registryPort) } - reg := tool.NewRegistry() + if os.Getenv("DISABLE_REGISTRY") == "" { + reg := tool.NewRegistry() + defer reg.Destroy() + } code := m.Run() - reg.Destroy() os.Exit(code) } diff --git a/smoke/tests/native_layer_test.go b/smoke/tests/native_layer_test.go index 77619a4aed8..8e72cb56a14 100644 --- a/smoke/tests/native_layer_test.go +++ b/smoke/tests/native_layer_test.go @@ -45,7 +45,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator { Dimension(paramEncrypt, []interface{}{false, true}). Dimension(paramAmplifyIO, []interface{}{uint64(0x100000)}). Skip(func(param *tool.DescartesItem) bool { - // rafs v6 not support cached mode nor dummy cache if param.GetString(paramFSVersion) == "6" { return param.GetString(paramRafsMode) == "cached" || param.GetString(paramCacheType) == "" @@ -70,17 +69,16 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator { } scenario := scenarios.Next() - ctx := tool.DefaultContext(n.t) - ctx.Build.Compressor = scenario.GetString(paramCompressor) - ctx.Build.FSVersion = scenario.GetString(paramFSVersion) - ctx.Build.ChunkSize = scenario.GetString(paramChunkSize) - ctx.Runtime.CacheType = scenario.GetString(paramCacheType) - ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) - ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) - ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) - ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) - return scenario.Str(), func(t *testing.T) { + ctx := tool.DefaultContext(n.t) + ctx.Build.Compressor = scenario.GetString(paramCompressor) + ctx.Build.FSVersion = scenario.GetString(paramFSVersion) + ctx.Build.ChunkSize = scenario.GetString(paramChunkSize) + ctx.Runtime.CacheType = scenario.GetString(paramCacheType) + ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) + ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) + ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) + ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) n.testMakeLayers(*ctx, t) } } @@ -105,7 +103,6 @@ func (n *NativeLayerTestSuite) TestAmplifyIO() test.Generator { /* Amplify io - target param */ Dimension(paramAmplifyIO, []interface{}{uint64(0x0), uint64(0x100000), uint64(0x10000000)}). Skip(func(param *tool.DescartesItem) bool { - // Rafs v6 not support cached mode nor dummy cache if param.GetString(paramFSVersion) == "6" { return param.GetString(paramRafsMode) == "cached" || param.GetString(paramCacheType) == "" @@ -130,17 +127,16 @@ func (n *NativeLayerTestSuite) TestAmplifyIO() test.Generator { } scenario := scenarios.Next() - ctx := tool.DefaultContext(n.t) - ctx.Build.Compressor = scenario.GetString(paramCompressor) - ctx.Build.FSVersion = scenario.GetString(paramFSVersion) - ctx.Build.ChunkSize = scenario.GetString(paramChunkSize) - ctx.Runtime.CacheType = scenario.GetString(paramCacheType) - ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) - ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) - ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) - ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) - return scenario.Str(), func(t *testing.T) { + ctx := tool.DefaultContext(n.t) + ctx.Build.Compressor = scenario.GetString(paramCompressor) + ctx.Build.FSVersion = scenario.GetString(paramFSVersion) + ctx.Build.ChunkSize = scenario.GetString(paramChunkSize) + ctx.Runtime.CacheType = scenario.GetString(paramCacheType) + ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) + ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) + ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) + ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) n.testMakeLayers(*ctx, t) } } diff --git a/smoke/tests/overlay_fs_test.go b/smoke/tests/overlay_fs_test.go index 818bfe93ecb..d3a92238aa4 100644 --- a/smoke/tests/overlay_fs_test.go +++ b/smoke/tests/overlay_fs_test.go @@ -51,6 +51,7 @@ func (ts *OverlayFsTestSuite) prepareTestEnv(t *testing.T) *tool.Context { // Verify lower layer mounted by nydusd ctx.Env.BootstrapPath = lowerBootstrap tool.Verify(t, *ctx, lowerLayer.FileTree) + return ctx } diff --git a/smoke/tests/tool/iterator.go b/smoke/tests/tool/iterator.go index 8622643df45..313ddb0dedc 100644 --- a/smoke/tests/tool/iterator.go +++ b/smoke/tests/tool/iterator.go @@ -2,10 +2,25 @@ package tool import ( "fmt" + "os" "sort" "strings" ) +func isIgnoredByEnv(param *DescartesItem) bool { + if skipCases := os.Getenv("SKIP_CASES"); skipCases != "" { + kvs := strings.Split(skipCases, ",") + for _, kv := range kvs { + k := strings.Split(kv, "=")[0] + v := strings.Split(kv, "=")[1] + if param.GetString(k) == v { + return true + } + } + } + return false +} + type DescartesItem struct { vals map[string]interface{} } @@ -141,7 +156,7 @@ func (c *DescartesIterator) calNext() { for name, idx := range c.cursorMap { item.vals[name] = c.valLists[idx][cursors[idx]] } - if c.skip == nil || !c.skip(item) { + if !isIgnoredByEnv(item) && (c.skip == nil || !c.skip(item)) { c.haveNext(cursors, item) return } diff --git a/smoke/tests/tool/layer.go b/smoke/tests/tool/layer.go index 523ea301198..9075f9e2188 100644 --- a/smoke/tests/tool/layer.go +++ b/smoke/tests/tool/layer.go @@ -232,6 +232,9 @@ func (l *Layer) recordFileTree(t *testing.T) { l.FileTree = map[string]*File{} filepath.Walk(l.workDir, func(path string, _ os.FileInfo, _ error) error { targetPath := l.TargetPath(t, path) + if targetPath == "." || targetPath == ".." { + return nil + } l.FileTree[targetPath] = NewFile(t, path, targetPath) return nil }) diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index d1f155309b0..9eec99fc812 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -10,15 +10,20 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "net" "net/http" "os" "os/exec" + "path/filepath" "strings" + "testing" "text/template" "time" + "github.com/google/uuid" "github.com/pkg/errors" + "github.com/stretchr/testify/require" ) type GlobalMetrics struct { @@ -69,6 +74,9 @@ type NydusdConfig struct { AccessPattern bool PrefetchFiles []string AmplifyIO uint64 + // Hot Upgrade config. + Upgrade bool + SupervisorSockPath string // Overlay config. OvlUpperDir string OvlWorkDir string @@ -76,6 +84,8 @@ type NydusdConfig struct { } type Nydusd struct { + client *http.Client + cmd *exec.Cmd NydusdConfig } @@ -168,8 +178,34 @@ func makeConfig(tplType TemplateType, conf NydusdConfig) error { return nil } -func CheckReady(ctx context.Context, sock string) <-chan bool { - ready := make(chan bool) +func newNydusd(conf NydusdConfig) (*Nydusd, error) { + args := []string{ + "--mountpoint", + conf.MountPath, + "--apisock", + conf.APISockPath, + "--log-level", + "error", + } + if len(conf.ConfigPath) > 0 { + args = append(args, "--config", conf.ConfigPath) + } + if len(conf.BootstrapPath) > 0 { + args = append(args, "--bootstrap", conf.BootstrapPath) + } + if conf.Upgrade { + args = append(args, "--upgrade") + } + if len(conf.SupervisorSockPath) > 0 { + args = append(args, "--supervisor", conf.SupervisorSockPath, "--id", uuid.NewString()) + } + if conf.Writable { + args = append(args, "--writable") + } + + cmd := exec.Command(conf.NydusdPath, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr transport := &http.Transport{ MaxIdleConns: 10, @@ -180,7 +216,7 @@ func CheckReady(ctx context.Context, sock string) <-chan bool { Timeout: 5 * time.Second, KeepAlive: 5 * time.Second, } - return dialer.DialContext(ctx, "unix", sock) + return dialer.DialContext(ctx, "unix", conf.APISockPath) }, } @@ -189,109 +225,96 @@ func CheckReady(ctx context.Context, sock string) <-chan bool { Transport: transport, } - go func() { - for { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := client.Get(fmt.Sprintf("http://unix%s", "/api/v1/daemon")) - if err != nil { - continue - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - continue - } - - var info daemonInfo - if err = json.Unmarshal(body, &info); err != nil { - continue - } - - if info.State == "RUNNING" { - ready <- true - break - } - } - }() + nydusd := &Nydusd{ + client: client, + cmd: cmd, + NydusdConfig: conf, + } - return ready + return nydusd, nil } func NewNydusd(conf NydusdConfig) (*Nydusd, error) { if err := makeConfig(NydusdConfigTpl, conf); err != nil { return nil, errors.Wrap(err, "create config file for Nydusd") } - return &Nydusd{ - NydusdConfig: conf, - }, nil + + nydusd, err := newNydusd(conf) + if err != nil { + return nil, err + } + + return nydusd, nil } func NewNydusdWithOverlay(conf NydusdConfig) (*Nydusd, error) { if err := makeConfig(NydusdOvlConfigTpl, conf); err != nil { return nil, errors.Wrap(err, "create config file for Nydusd") } - return &Nydusd{ - NydusdConfig: conf, - }, nil -} - -func (nydusd *Nydusd) Mount() error { - _ = nydusd.Umount() - args := []string{ - "--mountpoint", - nydusd.MountPath, - "--apisock", - nydusd.APISockPath, - "--log-level", - "error", + nydusd, err := newNydusd(conf) + if err != nil { + return nil, err } - if len(nydusd.ConfigPath) > 0 { - args = append(args, "--config", nydusd.ConfigPath) + + return nydusd, nil +} + +func NewNydusdWithContext(ctx Context) (*Nydusd, error) { + conf := NydusdConfig{ + EnablePrefetch: ctx.Runtime.EnablePrefetch, + NydusdPath: ctx.Binary.Nydusd, + BootstrapPath: ctx.Env.BootstrapPath, + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + BackendType: "localfs", + BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), + BlobCacheDir: ctx.Env.CacheDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + MountPath: filepath.Join(ctx.Env.WorkDir, "mnt"), + CacheType: ctx.Runtime.CacheType, + CacheCompressed: ctx.Runtime.CacheCompressed, + RafsMode: ctx.Runtime.RafsMode, + DigestValidate: false, + AmplifyIO: ctx.Runtime.AmplifyIO, } - if len(nydusd.BootstrapPath) > 0 { - args = append(args, "--bootstrap", nydusd.BootstrapPath) + + if err := makeConfig(NydusdConfigTpl, conf); err != nil { + return nil, errors.Wrap(err, "create config file for Nydusd") } - if nydusd.Writable { - args = append(args, "--writable") + + nydusd, err := newNydusd(conf) + if err != nil { + return nil, err } - cmd := exec.Command(nydusd.NydusdPath, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr + return nydusd, nil +} + +func (nydusd *Nydusd) Run() (chan error, error) { + errChan := make(chan error) + if err := nydusd.cmd.Start(); err != nil { + return errChan, err + } - runErr := make(chan error) go func() { - runErr <- cmd.Run() + errChan <- nydusd.cmd.Wait() }() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + time.Sleep(2 * time.Second) - ready := CheckReady(ctx, nydusd.APISockPath) + return errChan, nil +} - select { - case err := <-runErr: - if err != nil { - return errors.Wrap(err, "run Nydusd binary") - } - case <-ready: - return nil - case <-time.After(10 * time.Second): - return errors.New("timeout to wait Nydusd ready") +func (nydusd *Nydusd) Mount() error { + _, err := nydusd.Run() + if err != nil { + return err } - return nil + return nydusd.WaitStatus("RUNNING") } func (nydusd *Nydusd) MountByAPI(config NydusdConfig) error { - err := makeConfig(NydusdConfigTpl, config) if err != nil { return err @@ -318,28 +341,11 @@ func (nydusd *Nydusd) MountByAPI(config NydusdConfig) error { PrefetchFiles: config.PrefetchFiles, } - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - body, err := json.Marshal(nydusdConfig) if err != nil { return err } - _, err = client.Post( + _, err = nydusd.client.Post( fmt.Sprintf("http://unix/api/v1/mount?mountpoint=%s", config.MountPath), "application/json", bytes.NewBuffer(body), @@ -359,27 +365,91 @@ func (nydusd *Nydusd) Umount() error { return nil } -func (nydusd *Nydusd) GetGlobalMetrics() (*GlobalMetrics, error) { +func (nydusd *Nydusd) WaitStatus(states ...string) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, + var currentState string + + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout to wait nydusd state, expected: %s, current: %s", states, currentState) + default: + } + + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix%s", "/api/v1/daemon")) + if err != nil { + continue + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + continue + } + + var info daemonInfo + if err = json.Unmarshal(body, &info); err != nil { + continue + } + currentState = info.State + + for _, state := range states { + if currentState == state { + return nil } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, + } } +} - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, +func (nydusd *Nydusd) SendFd() error { + req, err := http.NewRequest("PUT", "http://unix/api/v1/daemon/fuse/sendfd", nil) + if err != nil { + return err + } + + resp, err := nydusd.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +func (nydusd *Nydusd) Takeover() error { + req, err := http.NewRequest("PUT", "http://unix/api/v1/daemon/fuse/takeover", nil) + if err != nil { + return err + } + + resp, err := nydusd.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +func (nydusd *Nydusd) Exit() error { + req, err := http.NewRequest("PUT", "http://unix/api/v1/daemon/exit", nil) + if err != nil { + return err } - resp, err := client.Get(fmt.Sprintf("http://unix%s", "/api/v1/metrics")) + resp, err := nydusd.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + +func (nydusd *Nydusd) GetGlobalMetrics() (*GlobalMetrics, error) { + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix%s", "/api/v1/metrics")) if err != nil { return nil, err } @@ -399,25 +469,7 @@ func (nydusd *Nydusd) GetGlobalMetrics() (*GlobalMetrics, error) { } func (nydusd *Nydusd) GetFilesMetrics(id string) (map[string]FileMetrics, error) { - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - - resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/files?id=%s", id)) + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix/api/v1/metrics/files?id=%s", id)) if err != nil { return nil, err } @@ -437,25 +489,7 @@ func (nydusd *Nydusd) GetFilesMetrics(id string) (map[string]FileMetrics, error) } func (nydusd *Nydusd) GetBackendMetrics(id string) (*BackendMetrics, error) { - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - - resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/backend?id=%s", id)) + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix/api/v1/metrics/backend?id=%s", id)) if err != nil { return nil, err } @@ -475,25 +509,7 @@ func (nydusd *Nydusd) GetBackendMetrics(id string) (*BackendMetrics, error) { } func (nydusd *Nydusd) GetLatestFileMetrics() ([][]uint64, error) { - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - - resp, err := client.Get("http://unix/api/v1/metrics/files?latest=true") + resp, err := nydusd.client.Get("http://unix/api/v1/metrics/files?latest=true") if err != nil { return nil, err } @@ -513,30 +529,12 @@ func (nydusd *Nydusd) GetLatestFileMetrics() ([][]uint64, error) { } func (nydusd *Nydusd) GetAccessPatternMetrics(id string) ([]AccessPatternMetrics, error) { - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - args := "" if len(id) > 0 { args += "?id=" + id } - resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/pattern%s", args)) + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix/api/v1/metrics/pattern%s", args)) if err != nil { return nil, err } @@ -560,31 +558,12 @@ func (nydusd *Nydusd) GetAccessPatternMetrics(id string) ([]AccessPatternMetrics } func (nydusd *Nydusd) GetBlobCacheMetrics(id string) (*BlobCacheMetrics, error) { - - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - args := "" if len(id) > 0 { args += "?id=" + id } - resp, err := client.Get(fmt.Sprintf("http://unix/api/v1/metrics/blobcache%s", args)) + resp, err := nydusd.client.Get(fmt.Sprintf("http://unix/api/v1/metrics/blobcache%s", args)) if err != nil { return nil, err } @@ -604,26 +583,7 @@ func (nydusd *Nydusd) GetBlobCacheMetrics(id string) (*BlobCacheMetrics, error) } func (nydusd *Nydusd) GetInflightMetrics() (*InflightMetrics, error) { - - transport := &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - dialer := &net.Dialer{ - Timeout: 5 * time.Second, - KeepAlive: 5 * time.Second, - } - return dialer.DialContext(ctx, "unix", nydusd.APISockPath) - }, - } - - client := &http.Client{ - Timeout: 30 * time.Second, - Transport: transport, - } - - resp, err := client.Get("http://unix/api/v1/metrics/inflight") + resp, err := nydusd.client.Get("http://unix/api/v1/metrics/inflight") if err != nil { return nil, err } @@ -645,3 +605,45 @@ func (nydusd *Nydusd) GetInflightMetrics() (*InflightMetrics, error) { return &info, err } + +func (nydusd *Nydusd) Verify(t *testing.T, expectedFileTree map[string]*File) { + actualFiles := map[string]*File{} + err := filepath.WalkDir(nydusd.MountPath, func(path string, _ fs.DirEntry, err error) error { + require.Nil(t, err) + + targetPath, err := filepath.Rel(nydusd.MountPath, path) + require.NoError(t, err) + + if targetPath == "." || targetPath == ".." { + return nil + } + + file := NewFile(t, path, targetPath) + actualFiles[targetPath] = file + if expectedFileTree[targetPath] != nil { + expectedFileTree[targetPath].Compare(t, file) + } else { + t.Fatalf("not found file %s in OCI layer", targetPath) + } + + return nil + }) + require.NoError(t, err) + + for targetPath, file := range expectedFileTree { + if actualFiles[targetPath] != nil { + actualFiles[targetPath].Compare(t, file) + } else { + t.Fatalf("not found file %s in nydus layer: %s %s", targetPath, nydusd.MountPath, nydusd.BootstrapPath) + } + } +} + +func Verify(t *testing.T, ctx Context, expectedFileTree map[string]*File) { + nydusd, err := NewNydusdWithContext(ctx) + require.NoError(t, err) + err = nydusd.Mount() + require.NoError(t, err) + defer nydusd.Umount() + nydusd.Verify(t, expectedFileTree) +} diff --git a/smoke/tests/tool/verify.go b/smoke/tests/tool/verify.go deleted file mode 100644 index b3dd177d373..00000000000 --- a/smoke/tests/tool/verify.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2023 Nydus Developers. All rights reserved. -// -// SPDX-License-Identifier: Apache-2.0 - -package tool - -import ( - "fmt" - "io/fs" - "path/filepath" - "testing" - - "github.com/containerd/log" - "github.com/stretchr/testify/require" -) - -func Verify(t *testing.T, ctx Context, expectedFiles map[string]*File) { - config := NydusdConfig{ - EnablePrefetch: ctx.Runtime.EnablePrefetch, - NydusdPath: ctx.Binary.Nydusd, - BootstrapPath: ctx.Env.BootstrapPath, - ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), - BackendType: "localfs", - BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), - BlobCacheDir: ctx.Env.CacheDir, - APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), - MountPath: ctx.Env.MountDir, - CacheType: ctx.Runtime.CacheType, - CacheCompressed: ctx.Runtime.CacheCompressed, - RafsMode: ctx.Runtime.RafsMode, - DigestValidate: false, - AmplifyIO: ctx.Runtime.AmplifyIO, - } - - nydusd, err := NewNydusd(config) - require.NoError(t, err) - err = nydusd.Mount() - require.NoError(t, err) - defer func() { - if err := nydusd.Umount(); err != nil { - log.L.WithError(err).Errorf("umount") - } - }() - - actualFiles := map[string]*File{} - err = filepath.WalkDir(ctx.Env.MountDir, func(path string, _ fs.DirEntry, err error) error { - require.Nil(t, err) - - targetPath, err := filepath.Rel(ctx.Env.MountDir, path) - require.NoError(t, err) - - file := NewFile(t, path, targetPath) - actualFiles[targetPath] = file - if expectedFiles[targetPath] != nil { - expectedFiles[targetPath].Compare(t, file) - } else { - t.Fatalf("not found file %s in OCI layer", targetPath) - } - - return nil - }) - require.NoError(t, err) - - for targetPath, file := range expectedFiles { - if actualFiles[targetPath] != nil { - actualFiles[targetPath].Compare(t, file) - } else { - t.Fatalf("not found file %s in nydus layer", targetPath) - } - } -} From 717999e8b4b39ada2fbd40e3e46e5726a378eef1 Mon Sep 17 00:00:00 2001 From: Yan Song Date: Thu, 18 Jul 2024 10:51:07 +0000 Subject: [PATCH 2/2] smoke: add nydusd hot upgrade test case The test case in hot_upgrade_test.go is different with takeover_test.go, it not depend on snapshotter component. Signed-off-by: Yan Song --- smoke/tests/hot_upgrade_test.go | 156 ++++++++++++++++++++++++++++++++ smoke/tests/tool/nydusd.go | 15 +++ 2 files changed, 171 insertions(+) create mode 100644 smoke/tests/hot_upgrade_test.go diff --git a/smoke/tests/hot_upgrade_test.go b/smoke/tests/hot_upgrade_test.go new file mode 100644 index 00000000000..dbfd141fb95 --- /dev/null +++ b/smoke/tests/hot_upgrade_test.go @@ -0,0 +1,156 @@ +// Copyright 2024 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/containerd/nydus-snapshotter/pkg/converter" + "github.com/containerd/nydus-snapshotter/pkg/supervisor" + "github.com/dragonflyoss/nydus/smoke/tests/texture" + "github.com/dragonflyoss/nydus/smoke/tests/tool" + "github.com/dragonflyoss/nydus/smoke/tests/tool/test" + "github.com/stretchr/testify/require" +) + +type Snapshotter struct { +} + +type HotUpgradeTestSuite struct { + t *testing.T +} + +func (c *HotUpgradeTestSuite) buildLayer(t *testing.T, ctx *tool.Context, rootFs *tool.Layer) string { + digest := rootFs.Pack(t, + converter.PackOption{ + BuilderPath: ctx.Binary.Builder, + Compressor: "lz4_block", + FsVersion: "5", + }, + ctx.Env.BlobDir) + _, bootstrap := tool.MergeLayers(t, *ctx, + converter.MergeOption{ + BuilderPath: ctx.Binary.Builder, + }, + []converter.Layer{ + {Digest: digest}, + }) + return bootstrap +} + +func (c *HotUpgradeTestSuite) newNydusd(t *testing.T, ctx *tool.Context, bootstrap, name string, upgrade bool) *tool.Nydusd { + config := tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, fmt.Sprintf("nydusd-api-%s.sock", name)), + ConfigPath: filepath.Join(ctx.Env.WorkDir, fmt.Sprintf("nydusd-config.fusedev-%s.json", name)), + SupervisorSockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-supervisor.sock"), + } + if upgrade { + config.Upgrade = true + } + + nydusd, err := tool.NewNydusd(config) + require.NoError(t, err) + + _, err = nydusd.Run() + require.NoError(t, err) + + if upgrade { + err = nydusd.WaitStatus("INIT") + } else { + err = nydusd.WaitStatus("RUNNING") + } + require.NoError(t, err) + + config.BootstrapPath = bootstrap + config.MountPath = "/" + config.BackendType = "localfs" + config.BackendConfig = fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir) + config.EnablePrefetch = true + config.PrefetchFiles = []string{"/"} + config.BlobCacheDir = ctx.Env.CacheDir + config.CacheType = ctx.Runtime.CacheType + config.CacheCompressed = ctx.Runtime.CacheCompressed + config.RafsMode = ctx.Runtime.RafsMode + + err = nydusd.MountByAPI(config) + require.NoError(t, err) + + return nydusd +} + +func (c *HotUpgradeTestSuite) TestHotUpgrade(t *testing.T) { + ctx := tool.DefaultContext(t) + ctx.PrepareWorkDir(t) + defer ctx.Destroy(t) + + // Build nydus layer + layer := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root")) + bootstrap := c.buildLayer(t, ctx, layer) + + // Start snapshotter simulator + ss, err := supervisor.NewSupervisorSet(filepath.Join(ctx.Env.WorkDir)) + require.NoError(t, err) + supervisor := ss.NewSupervisor("nydusd-supervisor") + defer ss.DestroySupervisor("nydusd-supervisor") + + // Start old nydusd to mount rootfs + oldNydusd := c.newNydusd(t, ctx, bootstrap, "old", false) + defer oldNydusd.Umount() + + // Old nydusd's state should be RUNNING + err = oldNydusd.WaitStatus("RUNNING") + require.NoError(t, err) + + // Verify filesytem on new nydusd + oldNydusd.Verify(t, layer.FileTree) + + // Snapshotter receive fuse fd from old nydusd + err = supervisor.FetchDaemonStates(oldNydusd.SendFd) + require.NoError(t, err) + + // Start new nydusd in upgrade mode (don't mount) + newNydusd := c.newNydusd(t, ctx, bootstrap, "new", true) + defer newNydusd.Umount() + + // New nydusd's state should be INIT + err = newNydusd.WaitStatus("INIT") + require.NoError(t, err) + + // Tells old nydusd to exit + err = oldNydusd.Exit() + require.NoError(t, err) + + // Send fuse fd to new nydusd + err = supervisor.SendStatesTimeout(time.Second * 5) + require.NoError(t, err) + err = newNydusd.Takeover() + require.NoError(t, err) + + // New nydusd's state should be RUNNING | READY + // Only have RUNNING state for older nydusd version (v1.x) + err = newNydusd.WaitStatus("RUNNING", "READY") + require.NoError(t, err) + + // Snapshotter receive fuse fd from new nydusd + err = supervisor.FetchDaemonStates(newNydusd.SendFd) + require.NoError(t, err) + + // Start new nydusd to serve mountpoint + // It's unnecessary for older nydusd version (v1.x) + err = newNydusd.StartByAPI() + require.NoError(t, err) + + // Verify filesytem on new nydusd + newNydusd.Verify(t, layer.FileTree) +} + +func TestHotUpgrade(t *testing.T) { + test.Run(t, &HotUpgradeTestSuite{t: t}) +} diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index 9eec99fc812..151154c4fb4 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -403,6 +403,21 @@ func (nydusd *Nydusd) WaitStatus(states ...string) error { } } +func (nydusd *Nydusd) StartByAPI() error { + req, err := http.NewRequest("PUT", "http://unix/api/v1/daemon/start", nil) + if err != nil { + return err + } + + resp, err := nydusd.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + return nil +} + func (nydusd *Nydusd) SendFd() error { req, err := http.NewRequest("PUT", "http://unix/api/v1/daemon/fuse/sendfd", nil) if err != nil {