nydusify: refine build subcommand

Improve help messages and fix a crash bug related to oss.

Signed-off-by: Jiang Liu <[email protected]>

contrib/nydusify/cmd/nydusify.go

@@ -441,99 +441,111 @@ func main() {
 			},
 		},
 		{
-			Name:  "pack",
-			Usage: "Pack a directory to nydus bootstrap and blob",
+			Name:    "build",
+			Aliases: []string{"pack"},
+			Usage:   "Build a Nydus image from a source directory",
 			Flags: []cli.Flag{
 				&cli.StringFlag{
 					Name:     "source-dir",
 					Aliases:  []string{"target-dir"}, // for compatibility
 					Required: true,
-					Usage:    "The source directory of build target",
+					Usage:    "Source directory to build image from",
 					EnvVars:  []string{"SOURCE_DIR"},
 				},
 				&cli.StringFlag{
 					Name:     "output-dir",
 					Aliases:  []string{"o"},
 					Required: false,
-					Usage:    "Output dir of build artifact",
+					Usage:    "Output directory for built artifacts",
 					EnvVars:  []string{"OUTPUT_DIR"},
 				},
+				&cli.StringFlag{
+					Name:     "name",
+					Aliases:  []string{"meta", "bootstrap"}, // for compatibility
+					Required: true,
+					Usage:    "Image name, which will be used as suffix for the generated Nydus image bootstrap/data blobs",
+					EnvVars:  []string{"BOOTSTRAP", "IMAGE_NAME"},
+				},
+
 				&cli.BoolFlag{
 					Name:    "backend-push",
 					Value:   false,
-					Usage:   "Push Nydus blob to storage backend",
+					Usage:   "Push generated Nydus image to storage backend",
 					EnvVars: []string{"BACKEND_PUSH"},
 				},
 				&cli.StringFlag{
 					Name:        "backend-type",
 					Value:       "oss",
 					DefaultText: "oss",
-					Usage:       "Specify Nydus blob storage backend type",
+					Usage:       "Type of storage backend, possible values: 'registry', 'oss'",
 					EnvVars:     []string{"BACKEND_TYPE"},
 				},
 				&cli.StringFlag{
-					Name:     "bootstrap",
-					Aliases:  []string{"meta"}, // for compatibility
-					Required: true,
-					Usage:    "Specify Nydus meta file name",
-					EnvVars:  []string{"BOOTSTRAP"},
-				},
-				&cli.StringFlag{
-					Name:    "parent-bootstrap",
-					Usage:   "Specify parent bootstrap to pack dictionary",
-					EnvVars: []string{"PARENT_BOOTSTRAP"},
-				},
-				&cli.StringFlag{
-					Name: "chunk-dict",
-					Usage: "Specify a chunk dict expression for chunk deduplication, " +
-						"for example: bootstrap=/path/to/dict.boot",
-					EnvVars: []string{"CHUNK_DICT"},
+					Name:    "backend-config",
+					Value:   "",
+					Usage:   "Json string for storage backend configuration",
+					EnvVars: []string{"BACKEND_CONFIG"},
 				},
 				&cli.StringFlag{
 					Name:      "backend-config-file",
 					TakesFile: true,
-					Usage:     "Specify Nydus blob storage backend config from path",
+					Usage:     "Json configuration file for storage backend",
 					EnvVars:   []string{"BACKEND_CONFIG_FILE"},
 				},
+
 				&cli.StringFlag{
-					Name:    "nydus-image",
-					Value:   "nydus-image",
-					Usage:   "The nydus-image binary path, if unset, search in PATH environment",
-					EnvVars: []string{"NYDUS_IMAGE"},
+					Name:    "chunk-dict",
+					Usage:   "Specify a chunk dict expression for chunk deduplication, for example: bootstrap=/path/to/dict.boot",
+					EnvVars: []string{"CHUNK_DICT"},
+				},
+				&cli.StringFlag{
+					Name:    "parent-bootstrap",
+					Usage:   "Specify a parent metadata to reference data chunks",
+					EnvVars: []string{"PARENT_BOOTSTRAP"},
 				},
 				&cli.BoolFlag{
 					Name:    "compact",
-					Usage:   "Compact parent bootstrap if necessary before do pack",
+					Usage:   "Compact parent bootstrap before building the image when needed",
 					EnvVars: []string{"COMPACT"},
 				},
 				&cli.StringFlag{
 					Name: "compact-config-file",
-					Usage: "Compact config file, default config is " +
+					Usage: "Compact configuration file, default configuration is " +
 						"{\"min_used_ratio\": 5, \"compact_blob_size\": 10485760, \"max_compact_size\": 104857600, " +
 						"\"layers_to_compact\": 32}",
 					EnvVars: []string{"COMPACT_CONFIG_FILE"},
 				},
+
 				&cli.StringFlag{
 					Name:        "fs-version",
 					Required:    false,
-					Usage:       "Version number of nydus image format, possible values: 5, 6",
+					Usage:       "Nydus image format version number, possible values: 5, 6",
 					EnvVars:     []string{"FS_VERSION"},
 					Value:       "5",
 					DefaultText: "V5 format",
 				},
+
+				&cli.StringFlag{
+					Name:    "nydus-image",
+					Value:   "nydus-image",
+					Usage:   "Path to the nydus-image binary, default to search in PATH",
+					EnvVars: []string{"NYDUS_IMAGE"},
+				},
 			},
 			Before: func(ctx *cli.Context) error {
-				targetPath := ctx.String("target-dir")
-				fi, err := os.Stat(targetPath)
+				sourcePath := ctx.String("source-dir")
+				fi, err := os.Stat(sourcePath)
 				if err != nil {
-					return errors.Wrapf(err, "failed to stat target path %s", targetPath)
+					return errors.Wrapf(err, "failed to check source directory")
 				}
 				if !fi.IsDir() {
-					return errors.Errorf("%s is not a directory", targetPath)
+					return errors.Errorf("source path '%s' is not a directory", sourcePath)
 				}
 				return nil
 			},
 			Action: func(c *cli.Context) error {
+				setupLogLevel(c)
+
 				var (
 					p             *packer.Packer
 					res           packer.PackResult
@@ -542,23 +554,22 @@ func main() {
 				)
 
 				// if backend-push is specified, we should make sure backend-config-file exists
-				if c.Bool("backend-push") {
-					backendConfigFile := c.String("backend-config-file")
-					if strings.TrimSpace(backendConfigFile) == "" {
-						return errors.New("backend-config-file is required when specify --backend-push")
-					}
-					if _, err = os.Stat(backendConfigFile); err != nil {
-						return errors.Errorf("can not find backend-config-file %s", backendConfigFile)
+				if c.Bool("backend-push") || c.Bool("compact") {
+					_backendConfig, err := parseBackendConfig(
+						c.String("backend-config"), c.String("backend-config-file"),
+					)
+					if err != nil {
+						return err
+					} else if len(_backendConfig) == 0 {
+						return errors.Errorf("missing backend configuration information")
 					}
-					cfg, err := packer.ParseBackendConfig(backendConfigFile)
+					cfg, err := packer.ParseBackendConfig(_backendConfig)
 					if err != nil {
-						return errors.Errorf("failed to parse backend-config-file %s, err = %v", backendConfigFile, err)
+						return errors.Errorf("failed to parse backend-config '%s', err = %v", _backendConfig, err)
 					}
 					backendConfig = &cfg
 				}
 
-				setupLogLevel(c)
-
 				if p, err = packer.New(packer.Opt{
 					LogLevel:       logrus.GetLevel(),
 					NydusImagePath: c.String("nydus-image"),
@@ -569,27 +580,27 @@ func main() {
 				}
 
 				if res, err = p.Pack(context.Background(), packer.PackRequest{
-					Parent:    c.String("parent-bootstrap"),
-					ChunkDict: c.String("chunk-dict"),
-					TargetDir: c.String("target-dir"),
-					Meta:      c.String("bootstrap"),
-					PushBlob:  c.Bool("backend-push"),
+					SourceDir:    c.String("source-dir"),
+					ImageName:    c.String("name"),
+					PushToRemote: c.Bool("backend-push"),
+					FsVersion:    c.String("fs-version"),
 
+					ChunkDict:         c.String("chunk-dict"),
+					Parent:            c.String("parent-bootstrap"),
 					TryCompact:        c.Bool("compact"),
 					CompactConfigPath: c.String("compact-config-file"),
-					FsVersion:         c.String("fs-version"),
 				}); err != nil {
 					return err
 				}
-				logrus.Infof("successfully pack meta %s, blob %s", res.Meta, res.Blob)
+				logrus.Infof("successfully built Nydus image (bootstrap:'%s', blob:'%s')", res.Meta, res.Blob)
 				return nil
 			},
 		},
 	}
 
-	// Under platform linux/arm64, containerd/compression prioritizes using `unpigz`
-	// to decompress tar.giz, which will be corrupted somehow. By disabling it,
-	// keep nydusify behavior the same with x86_64 platform.
+	// With linux/arm64 platform, containerd/compression prioritizes `unpigz`
+	// to decompress tar.gz file, which may generate corrupted data somehow.
+	// Keep the same behavior with x86_64 platform by disabling pigz.
 	os.Setenv("CONTAINERD_DISABLE_PIGZ", "1")
 
 	if !utils.IsSupportedArch(runtime.GOARCH) {

contrib/nydusify/pkg/backend/oss.go

@@ -49,8 +49,8 @@ func newOSSBackend(rawConfig []byte) (*OSSBackend, error) {
 	accessKeySecret := configMap["access_key_secret"]
 	objectPrefix := configMap["object_prefix"]
 
-	if !ok1 || !ok2 {
-		return nil, fmt.Errorf("no endpoint or bucket is specified")
+	if !ok1 || endpoint == "" || !ok2 || bucketName == "" {
+		return nil, fmt.Errorf("invalid OSS configuration: missing 'endpoint' or 'bucket'")
 	}
 
 	client, err := oss.New(endpoint, accessKeyID, accessKeySecret)

contrib/nydusify/pkg/compactor/compactor.go

@@ -27,24 +27,24 @@ type CompactConfig struct {
 func (cfg *CompactConfig) Dumps(filePath string) error {
 	file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 	if err != nil {
-		return errors.Wrap(err, "open file failed")
+		return errors.Wrap(err, "failed to open file")
 	}
 	defer file.Close()
 	if err = json.NewEncoder(file).Encode(cfg); err != nil {
-		return errors.Wrap(err, "encode json failed")
+		return errors.Wrap(err, "failed to encode json")
 	}
 	return nil
 }
 
 func loadCompactConfig(filePath string) (CompactConfig, error) {
 	file, err := os.Open(filePath)
 	if err != nil {
-		return CompactConfig{}, errors.Wrap(err, "load compact config file failed")
+		return CompactConfig{}, errors.Wrap(err, "failed to load compact configuration file")
 	}
 	defer file.Close()
 	var cfg CompactConfig
 	if err = json.NewDecoder(file).Decode(&cfg); err != nil {
-		return CompactConfig{}, errors.Wrap(err, "decode compact config file failed")
+		return CompactConfig{}, errors.Wrap(err, "failed to decode compact configuration file")
 	}
 	return cfg, nil
 }
@@ -79,7 +79,7 @@ func NewCompactor(nydusImagePath, workdir, configPath string) (*Compactor, error
 func (compactor *Compactor) Compact(bootstrapPath, chunkDict, backendType, backendConfigFile string) (string, error) {
 	targetBootstrap := bootstrapPath + ".compact"
 	if err := os.Remove(targetBootstrap); err != nil && !os.IsNotExist(err) {
-		return "", errors.Wrap(err, "delete old target bootstrap failed")
+		return "", errors.Wrap(err, "failed to delete old bootstrap file")
 	}
 	// prepare config file
 	configFilePath := filepath.Join(compactor.workdir, "compact.json")
@@ -88,7 +88,7 @@ func (compactor *Compactor) Compact(bootstrapPath, chunkDict, backendType, backe
 	}
 	outputJSONPath := filepath.Join(compactor.workdir, "compact-result.json")
 	if err := os.Remove(outputJSONPath); err != nil && !os.IsNotExist(err) {
-		return "", errors.Wrap(err, "delete old output-json file failed")
+		return "", errors.Wrap(err, "failed to delete old output-json file")
 	}
 	err := compactor.builder.Compact(build.CompactOption{
 		ChunkDict:           chunkDict,
@@ -100,7 +100,7 @@ func (compactor *Compactor) Compact(bootstrapPath, chunkDict, backendType, backe
 		CompactConfigPath:   configFilePath,
 	})
 	if err != nil {
-		return "", errors.Wrap(err, "run compact command failed")
+		return "", errors.Wrap(err, "failed to run compact command")
 	}
 
 	return targetBootstrap, nil

contrib/nydusify/pkg/packer/artifact.go

@@ -3,6 +3,7 @@ package packer
 import (
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
 )
@@ -19,16 +20,25 @@ func NewArtifact(outputDir string) (Artifact, error) {
 	return res, nil
 }
 
-func (a Artifact) bootstrapPath(metaFileName string) string {
-	return filepath.Join(a.OutputDir, metaFileName)
+func (a Artifact) bootstrapPath(imageName string) string {
+	if filepath.Ext(imageName) != "" {
+		return filepath.Join(a.OutputDir, imageName)
+	}
+	return filepath.Join(a.OutputDir, imageName+".meta")
 }
 
-func (a Artifact) outputJSONPath() string {
-	return filepath.Join(a.OutputDir, "output.json")
+func (a Artifact) blobFilePath(imageName string, isDigest bool) string {
+	if isDigest {
+		return filepath.Join(a.OutputDir, imageName)
+	} else if suffix := filepath.Ext(imageName); suffix != "" {
+		return filepath.Join(a.OutputDir, strings.TrimSuffix(imageName, suffix)+".blob")
+	} else {
+		return filepath.Join(a.OutputDir, imageName+".blob")
+	}
 }
 
-func (a Artifact) blobFilePath(blobFileName string) string {
-	return filepath.Join(a.OutputDir, blobFileName)
+func (a Artifact) outputJSONPath() string {
+	return filepath.Join(a.OutputDir, "output.json")
 }
 
 // ensureOutputDir use user defined outputDir or defaultOutputDir, and make sure dir exists

contrib/nydusify/pkg/packer/artifact_test.go

@@ -0,0 +1,20 @@
+package packer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestArtifactPath(t *testing.T) {
+	artifact, err := NewArtifact("/tmp")
+
+	assert.Nil(t, err)
+	assert.Equal(t, artifact.bootstrapPath("test.meta"), "/tmp/test.meta")
+	assert.Equal(t, artifact.bootstrapPath("test.m"), "/tmp/test.m")
+	assert.Equal(t, artifact.bootstrapPath("test"), "/tmp/test.meta")
+	assert.Equal(t, artifact.blobFilePath("test.meta", false), "/tmp/test.blob")
+	assert.Equal(t, artifact.blobFilePath("test.m", false), "/tmp/test.blob")
+	assert.Equal(t, artifact.blobFilePath("test", false), "/tmp/test.blob")
+	assert.Equal(t, artifact.blobFilePath("test", true), "/tmp/test")
+}