Merge pull request #787 from giuseppe/big-data-layers

layers: support BigData

cmd/containers-storage/layer.go

@@ -3,12 +3,85 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/mflag"
 )
 
+var (
+	paramLayerDataFile = ""
+)
+
+func listLayerBigData(flags *mflag.FlagSet, action string, m storage.Store, args []string) int {
+	layer, err := m.Layer(args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	d, err := m.ListLayerBigData(layer.ID)
+	if jsonOutput {
+		json.NewEncoder(os.Stdout).Encode(d)
+	} else {
+		for _, name := range d {
+			fmt.Printf("%s\n", name)
+		}
+	}
+	return 0
+}
+
+func getLayerBigData(flags *mflag.FlagSet, action string, m storage.Store, args []string) int {
+	layer, err := m.Layer(args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	output := os.Stdout
+	if paramLayerDataFile != "" {
+		f, err := os.Create(paramLayerDataFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			return 1
+		}
+		output = f
+	}
+	b, err := m.LayerBigData(layer.ID, args[1])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	if _, err := io.Copy(output, b); err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	output.Close()
+	return 0
+}
+
+func setLayerBigData(flags *mflag.FlagSet, action string, m storage.Store, args []string) int {
+	layer, err := m.Layer(args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	input := os.Stdin
+	if paramLayerDataFile != "" {
+		f, err := os.Open(paramLayerDataFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+			return 1
+		}
+		input = f
+	}
+	err = m.SetLayerBigData(layer.ID, args[1], input)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%+v\n", err)
+		return 1
+	}
+	return 0
+}
+
 func layer(flags *mflag.FlagSet, action string, m storage.Store, args []string) int {
 	matched := []*storage.Layer{}
 	for _, arg := range args {
@@ -106,5 +179,36 @@ func init() {
 				flags.BoolVar(&jsonOutput, []string{"-json", "j"}, jsonOutput, "Prefer JSON output")
 			},
 		},
+		command{
+			names:       []string{"list-layer-data", "listlayerdata"},
+			optionsHelp: "[options [...]] layerNameOrID",
+			usage:       "List data items that are attached to a layer",
+			action:      listLayerBigData,
+			minArgs:     1,
+			maxArgs:     1,
+			addFlags: func(flags *mflag.FlagSet, cmd *command) {
+				flags.BoolVar(&jsonOutput, []string{"-json", "j"}, jsonOutput, "Prefer JSON output")
+			},
+		},
+		command{
+			names:       []string{"get-layer-data", "getlayerdata"},
+			optionsHelp: "[options [...]] layerID dataName",
+			usage:       "Get data that is attached to a layer",
+			action:      getLayerBigData,
+			minArgs:     2,
+			addFlags: func(flags *mflag.FlagSet, cmd *command) {
+				flags.StringVar(&paramLayerDataFile, []string{"-file", "f"}, paramLayerDataFile, "Write data to file")
+			},
+		},
+		command{
+			names:       []string{"set-layer-data", "setlayerdata"},
+			optionsHelp: "[options [...]] layerID dataName",
+			usage:       "Set data that is attached to a layer",
+			action:      setLayerBigData,
+			minArgs:     2,
+			addFlags: func(flags *mflag.FlagSet, cmd *command) {
+				flags.StringVar(&paramLayerDataFile, []string{"-file", "f"}, paramLayerDataFile, "Read data from file")
+			},
+		},
 	)
 }

docs/containers-storage-get-layer-data.md

@@ -0,0 +1,21 @@
+## containers-storage-get-layer-data 1 "December 2020"
+
+## NAME
+containers-storage get-layer-data - Retrieve lookaside data for a layer
+
+## SYNOPSIS
+**containers-storage** **get-layer-data** [*options* [...]] *layerID* *dataName*
+
+## DESCRIPTION
+Retrieves a piece of named data which is associated with a layer.
+
+## OPTIONS
+**-f | --file** *file*
+
+Write the data to a file instead of stdout.
+
+## EXAMPLE
+**containers-storage get-layer-data -f config.json 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 configuration**
+
+## SEE ALSO
+containers-storage-set-layer-data(1)

docs/containers-storage-list-layer-data.md

@@ -0,0 +1,18 @@
+## containers-storage-list-layer-data 1 "December 2020"
+
+## NAME
+containers-storage list-layer-data - List lookaside data for an layer
+
+## SYNOPSIS
+**containers-storage** **list-layer-data** *layerID*
+
+## DESCRIPTION
+List the pieces of named data which are associated with an layer.
+
+## EXAMPLE
+**containers-storage list-layer-data 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824**
+
+## SEE ALSO
+containers-storage-get-layer-data(1)
+containers-storage-get-layer-data-digest(1)
+containers-storage-set-layer-data(1)

docs/containers-storage-set-layer-data.md

@@ -0,0 +1,21 @@
+## containers-storage-set-layer-data 1 "December 2020"
+
+## NAME
+containers-storage set-layer-data - Set lookaside data for a layer
+
+## SYNOPSIS
+**containers-storage** **set-layer-data** [*options* [...]] *layerID* *dataName*
+
+## DESCRIPTION
+Sets a piece of named data which is associated with a layer.
+
+## OPTIONS
+**-f | --file** *filename*
+
+Read the data contents from a file instead of stdin.
+
+## EXAMPLE
+**containers-storage set-layer-data -f ./config.json 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 configuration**
+
+## SEE ALSO
+containers-storage-get-layer-data(1)

layers.go

@@ -117,6 +117,11 @@ type Layer struct {
 
 	// ReadOnly is true if this layer resides in a read-only layer store.
 	ReadOnly bool `json:"-"`
+
+	// BigDataNames is a list of names of data items that we keep for the
+	// convenience of the caller.  They can be large, and are only in
+	// memory when being read from or written to disk.
+	BigDataNames []string `json:"big-data-names,omitempty"`
 }
 
 type layerMountPoint struct {
@@ -137,6 +142,7 @@ type DiffOptions struct {
 type ROLayerStore interface {
 	ROFileBasedStore
 	ROMetadataStore
+	ROLayerBigDataStore
 
 	// Exists checks if a layer with the specified name or ID is known.
 	Exists(id string) bool
@@ -194,6 +200,7 @@ type LayerStore interface {
 	RWFileBasedStore
 	RWMetadataStore
 	FlaggableStore
+	RWLayerBigDataStore
 
 	// Create creates a new layer, optionally giving it a specified ID rather than
 	// a randomly-generated one, either inheriting data from another specified
@@ -278,6 +285,7 @@ func copyLayer(l *Layer) *Layer {
 		UncompressedSize:   l.UncompressedSize,
 		CompressionType:    l.CompressionType,
 		ReadOnly:           l.ReadOnly,
+		BigDataNames:       copyStringSlice(l.BigDataNames),
 		Flags:              copyStringInterfaceMap(l.Flags),
 		UIDMap:             copyIDMap(l.UIDMap),
 		GIDMap:             copyIDMap(l.GIDMap),
@@ -694,14 +702,15 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab
 	}
 	if err == nil {
 		layer = &Layer{
-			ID:         id,
-			Parent:     parent,
-			Names:      names,
-			MountLabel: mountLabel,
-			Created:    time.Now().UTC(),
-			Flags:      make(map[string]interface{}),
-			UIDMap:     copyIDMap(moreOptions.UIDMap),
-			GIDMap:     copyIDMap(moreOptions.GIDMap),
+			ID:           id,
+			Parent:       parent,
+			Names:        names,
+			MountLabel:   mountLabel,
+			Created:      time.Now().UTC(),
+			Flags:        make(map[string]interface{}),
+			UIDMap:       copyIDMap(moreOptions.UIDMap),
+			GIDMap:       copyIDMap(moreOptions.GIDMap),
+			BigDataNames: []string{},
 		}
 		r.layers = append(r.layers, layer)
 		r.idindex.Add(id)
@@ -970,6 +979,80 @@ func (r *layerStore) SetNames(id string, names []string) error {
 	return ErrLayerUnknown
 }
 
+func (r *layerStore) datadir(id string) string {
+	return filepath.Join(r.layerdir, id)
+}
+
+func (r *layerStore) datapath(id, key string) string {
+	return filepath.Join(r.datadir(id), makeBigDataBaseName(key))
+}
+
+func (r *layerStore) BigData(id, key string) (io.ReadCloser, error) {
+	if key == "" {
+		return nil, errors.Wrapf(ErrInvalidBigDataName, "can't retrieve layer big data value for empty name")
+	}
+	layer, ok := r.lookup(id)
+	if !ok {
+		return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id)
+	}
+	return os.Open(r.datapath(layer.ID, key))
+}
+
+func (r *layerStore) SetBigData(id, key string, data io.Reader) error {
+	if key == "" {
+		return errors.Wrapf(ErrInvalidBigDataName, "can't set empty name for layer big data item")
+	}
+	if !r.IsReadWrite() {
+		return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to save data items associated with layers at %q", r.layerspath())
+	}
+	layer, ok := r.lookup(id)
+	if !ok {
+		return errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q to write bigdata", id)
+	}
+	err := os.MkdirAll(r.datadir(layer.ID), 0700)
+	if err != nil {
+		return err
+	}
+
+	// NewAtomicFileWriter doesn't overwrite/truncate the existing inode.
+	// BigData() relies on this behaviour when opening the file for read
+	// so that it is either accessing the old data or the new one.
+	writer, err := ioutils.NewAtomicFileWriter(r.datapath(layer.ID, key), 0600)
+	if err != nil {
+		return errors.Wrapf(err, "error opening bigdata file")
+	}
+
+	if _, err := io.Copy(writer, data); err != nil {
+		writer.Close()
+		return errors.Wrapf(err, "error copying bigdata for the layer")
+
+	}
+	if err := writer.Close(); err != nil {
+		return errors.Wrapf(err, "error closing bigdata file for the layer")
+	}
+
+	addName := true
+	for _, name := range layer.BigDataNames {
+		if name == key {
+			addName = false
+			break
+		}
+	}
+	if addName {
+		layer.BigDataNames = append(layer.BigDataNames, key)
+		return r.Save()
+	}
+	return nil
+}
+
+func (r *layerStore) BigDataNames(id string) ([]string, error) {
+	layer, ok := r.lookup(id)
+	if !ok {
+		return nil, errors.Wrapf(ErrImageUnknown, "error locating layer with ID %q to retrieve bigdata names", id)
+	}
+	return copyStringSlice(layer.BigDataNames), nil
+}
+
 func (r *layerStore) Metadata(id string) (string, error) {
 	if layer, ok := r.lookup(id); ok {
 		return layer.Metadata, nil
@@ -1004,6 +1087,7 @@ func (r *layerStore) deleteInternal(id string) error {
 	err := r.driver.Remove(id)
 	if err == nil {
 		os.Remove(r.tspath(id))
+		os.RemoveAll(r.datadir(id))
 		delete(r.byid, id)
 		for _, name := range layer.Names {
 			delete(r.byname, name)