From 6099fda42c0b912fe0a908d5a9b41e66231f0ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 5 Jul 2023 19:04:31 +0800 Subject: [PATCH] feat: e2e for amplify io larger than fuse buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- smoke/Makefile | 2 +- smoke/tests/api_test.go | 2 +- smoke/tests/image_test.go | 7 ++-- smoke/tests/native_layer_test.go | 62 ++++++++++++++++++++++++++++++++ smoke/tests/tool/context.go | 2 ++ smoke/tests/tool/iterator.go | 4 +++ smoke/tests/tool/nydusd.go | 4 ++- smoke/tests/tool/verify.go | 1 + 8 files changed, 78 insertions(+), 6 deletions(-) diff --git a/smoke/Makefile b/smoke/Makefile index bb7318ad5fd..96260e5c8d4 100644 --- a/smoke/Makefile +++ b/smoke/Makefile @@ -15,7 +15,7 @@ build: # NYDUS_NYDUSIFY=/path/to/latest/nydusify \ # make test test: build - sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS) + sudo -E ./smoke.test -test.v -test.timeout 10m -test.parallel=16 -test.run=$(TESTS) # WORK_DIR=/tmp \ # NYDUS_BUILDER=/path/to/latest/nydus-image \ diff --git a/smoke/tests/api_test.go b/smoke/tests/api_test.go index 48bd598b4f2..a05ac84f86c 100644 --- a/smoke/tests/api_test.go +++ b/smoke/tests/api_test.go @@ -195,7 +195,7 @@ func (a *APIV1TestSuite) TestPrefetch(t *testing.T) { config.RafsMode = ctx.Runtime.RafsMode err = nydusd.MountByAPI(config) require.NoError(t, err) - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 15) bcm, err := nydusd.GetBlobCacheMetrics("") require.NoError(t, err) diff --git a/smoke/tests/image_test.go b/smoke/tests/image_test.go index ff785b38df5..c7c9e730dfb 100644 --- a/smoke/tests/image_test.go +++ b/smoke/tests/image_test.go @@ -15,9 +15,10 @@ import ( ) const ( - paramZran = "zran" - paramBatch = "batch" - paramEncrypt = "encrypt" + paramZran = "zran" + paramBatch = "batch" + paramEncrypt = "encrypt" + paramAmplifyIO = "amplify_io" ) type ImageTestSuite struct { diff --git a/smoke/tests/native_layer_test.go b/smoke/tests/native_layer_test.go index 99f1031b309..2c32eee30f9 100644 --- a/smoke/tests/native_layer_test.go +++ b/smoke/tests/native_layer_test.go @@ -43,6 +43,7 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator { Dimension(paramEnablePrefetch, []interface{}{false, true}). Dimension(paramBatch, []interface{}{"0", "0x100000"}). Dimension(paramEncrypt, []interface{}{false, true}). + Dimension(paramAmplifyIO, []interface{}{uint64(0x100000)}). Skip(func(param *tool.DescartesItem) bool { // rafs v6 not support cached mode nor dummy cache @@ -77,6 +78,67 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator { ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) + ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) + + return scenario.Str(), func(t *testing.T) { + n.testMakeLayers(*ctx, t) + } + } +} + +func (n *NativeLayerTestSuite) TestAmplifyIO() test.Generator { + + scenarios := tool.DescartesIterator{} + scenarios. + + /* Common params */ + Dimension(paramCompressor, []interface{}{"lz4_block"}). + Dimension(paramFSVersion, []interface{}{"5", "6"}). + Dimension(paramChunkSize, []interface{}{"0x100000"}). + Dimension(paramCacheType, []interface{}{"blobcache"}). + Dimension(paramCacheCompressed, []interface{}{false}). + Dimension(paramRafsMode, []interface{}{"direct"}). + Dimension(paramEnablePrefetch, []interface{}{true}). + Dimension(paramBatch, []interface{}{"0x100000"}). + Dimension(paramEncrypt, []interface{}{false}). + + /* Amplify io - target param */ + Dimension(paramAmplifyIO, []interface{}{uint64(0x0), uint64(0x100000), uint64(0x10000000)}). + Skip(func(param *tool.DescartesItem) bool { + + // Rafs v6 not support cached mode nor dummy cache + if param.GetString(paramFSVersion) == "6" { + return param.GetString(paramRafsMode) == "cached" || param.GetString(paramCacheType) == "" + } + + // Dummy cache not support prefetch + if param.GetString(paramCacheType) == "" && param.GetBool(paramEnablePrefetch) { + return true + } + + // Batch or encrypt not work with rafs v5. + if param.GetString(paramFSVersion) == "5" && (param.GetString(paramBatch) != "0" || param.GetBool(paramEncrypt)) { + return true + } + + return false + }) + + return func() (name string, testCase test.Case) { + if !scenarios.HasNext() { + return + } + scenario := scenarios.Next() + + ctx := tool.DefaultContext(n.t) + ctx.Build.Compressor = scenario.GetString(paramCompressor) + ctx.Build.FSVersion = scenario.GetString(paramFSVersion) + ctx.Build.ChunkSize = scenario.GetString(paramChunkSize) + ctx.Runtime.CacheType = scenario.GetString(paramCacheType) + ctx.Runtime.CacheCompressed = scenario.GetBool(paramCacheCompressed) + ctx.Runtime.RafsMode = scenario.GetString(paramRafsMode) + ctx.Runtime.EnablePrefetch = scenario.GetBool(paramEnablePrefetch) + ctx.Runtime.AmplifyIO = scenario.GetUInt64(paramAmplifyIO) return scenario.Str(), func(t *testing.T) { n.testMakeLayers(*ctx, t) diff --git a/smoke/tests/tool/context.go b/smoke/tests/tool/context.go index 9e714eb0cf0..703613be264 100644 --- a/smoke/tests/tool/context.go +++ b/smoke/tests/tool/context.go @@ -39,6 +39,7 @@ type RuntimeContext struct { CacheCompressed bool RafsMode string EnablePrefetch bool + AmplifyIO uint64 } type EnvContext struct { @@ -75,6 +76,7 @@ func DefaultContext(t *testing.T) *Context { CacheCompressed: false, RafsMode: "direct", EnablePrefetch: true, + AmplifyIO: uint64(0x100000), }, } } diff --git a/smoke/tests/tool/iterator.go b/smoke/tests/tool/iterator.go index 37f6e7310f8..bf49e32ef1e 100644 --- a/smoke/tests/tool/iterator.go +++ b/smoke/tests/tool/iterator.go @@ -47,6 +47,10 @@ func (d *DescartesItem) Str() string { return sb.String() } +func (d *DescartesItem) GetUInt64(name string) uint64 { + return d.vals[name].(uint64) +} + // Generator of Cartesian product. // // An example is below: diff --git a/smoke/tests/tool/nydusd.go b/smoke/tests/tool/nydusd.go index ee822d1b9d9..adfae105743 100644 --- a/smoke/tests/tool/nydusd.go +++ b/smoke/tests/tool/nydusd.go @@ -69,6 +69,7 @@ type NydusdConfig struct { LatestReadFiles bool AccessPattern bool PrefetchFiles []string + AmplifyIO uint64 } type Nydusd struct { @@ -104,7 +105,8 @@ var configTpl = ` "digest_validate": {{.DigestValidate}}, "enable_xattr": true, "latest_read_files": {{.LatestReadFiles}}, - "access_pattern": {{.AccessPattern}} + "access_pattern": {{.AccessPattern}}, + "amplify_io": {{.AmplifyIO}} } ` diff --git a/smoke/tests/tool/verify.go b/smoke/tests/tool/verify.go index 8d05cf2a66e..d2d4ad277d2 100644 --- a/smoke/tests/tool/verify.go +++ b/smoke/tests/tool/verify.go @@ -29,6 +29,7 @@ func Verify(t *testing.T, ctx Context, expectedFiles map[string]*File) { CacheCompressed: ctx.Runtime.CacheCompressed, RafsMode: ctx.Runtime.RafsMode, DigestValidate: false, + AmplifyIO: ctx.Runtime.AmplifyIO, } nydusd, err := NewNydusd(config)