Skip to content

Commit

Permalink
containers store: handle volatile and transient containers
Browse files Browse the repository at this point in the history 
This splits up the containers.json file in the containers store into
two, adding the new file `volatile-containers.json`. This new file is
saved using the NoSync options, which is faster but isn't robust in
the case of an unclean shutdown.

In the standard case, only containers marked as "volatile" (i.e. those
started with `--rm`) are stored in the volatile json file. This means
such containers are faster, but may get lost in case of an unclean
shutdown. This is fine for these containers though, as they are not
meant to persist.

In the transient store case, all containers are stored in the volatile
json file, and it (plus the matching lock file) is stored on runroot
(i.e. tmpfs) instead of the regular directory. This mean all
containers are fast to write, and none are persisted across boot.

Signed-off-by: Alexander Larsson <[email protected]>
  • Loading branch information
@alexlarsson
alexlarsson committed Nov 10, 2022
1 parent 47a09bb commit 9beea58
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 43 deletions.
158 changes: 119 additions & 39 deletions containers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,22 @@ import (
digest "github.com/opencontainers/go-digest"
)

type containerLocations uint8

// The backing store is split in two json file, one (the volatile)
// that is written without fsync() meaning it isn't as robust to
// unclean shutdown
const (
stableContainerLocation containerLocations = 1 << iota
volatileContainerLocation

numContainerLocationIndex = iota
)

func containerLocationFromIndex(index int) containerLocations {
return 1 << index
}

// A Container is a reference to a read-write layer with metadata.
type Container struct {
// ID is either one which was specified at create-time, or a random
Expand Down Expand Up @@ -64,6 +80,9 @@ type Container struct {
GIDMap []idtools.IDMap `json:"gidmap,omitempty"`

Flags map[string]interface{} `json:"flags,omitempty"`

// volatileStore is true if the container is from the volatile json file
volatileStore bool `json:"-"`
}

// rwContainerStore provides bookkeeping for information about Containers.
Expand Down Expand Up @@ -120,6 +139,7 @@ type rwContainerStore interface {
type containerStore struct {
lockfile Locker
dir string
jsonPath [numContainerLocationIndex]string
containers []*Container
idindex *truncindex.TruncIndex
byid map[string]*Container
Expand All @@ -142,6 +162,7 @@ func copyContainer(c *Container) *Container {
UIDMap: copyIDMap(c.UIDMap),
GIDMap: copyIDMap(c.GIDMap),
Flags: copyStringInterfaceMap(c.Flags),
volatileStore: c.volatileStore,
}
}

Expand Down Expand Up @@ -176,6 +197,13 @@ func (c *Container) MountOpts() []string {
}
}

func containerLocation(c *Container) containerLocations {
if c.volatileStore {
return volatileContainerLocation
}
return stableContainerLocation
}

// startWritingWithReload makes sure the store is fresh if canReload, and locks it for writing.
// If this succeeds, the caller MUST call stopWriting().
//
Expand Down Expand Up @@ -258,10 +286,6 @@ func (r *containerStore) Containers() ([]Container, error) {
return containers, nil
}

func (r *containerStore) containerspath() string {
return filepath.Join(r.dir, "containers.json")
}

func (r *containerStore) datadir(id string) string {
return filepath.Join(r.dir, id)
}
Expand All @@ -275,31 +299,51 @@ func (r *containerStore) datapath(id, key string) string {
// The caller must hold r.lockfile for reading _or_ writing; lockedForWriting is true
// if it is held for writing.
func (r *containerStore) load(lockedForWriting bool) error {
needSave := false
rpath := r.containerspath()
data, err := os.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}

var needSave containerLocations
containers := []*Container{}
if len(data) != 0 {
if err := json.Unmarshal(data, &containers); err != nil {
return fmt.Errorf("loading %q: %w", rpath, err)

ids := make(map[string]*Container)

for locationIndex := 0; locationIndex < numContainerLocationIndex; locationIndex++ {
location := containerLocationFromIndex(locationIndex)
rpath := r.jsonPath[locationIndex]
data, err := os.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}

locationContainers := []*Container{}

if len(data) != 0 {
if err := json.Unmarshal(data, &locationContainers); err != nil {
return fmt.Errorf("loading %q: %w", rpath, err)
}
}

for _, container := range locationContainers {
// There should be no duplicated ids between json files, but lets check to be sure
if ids[container.ID] != nil {
continue // skip invalid duplicated container
}
// Remember where the container came from
if location == volatileContainerLocation {
container.volatileStore = true
}
containers = append(containers, container)
ids[container.ID] = container
}
}

idlist := make([]string, 0, len(containers))
layers := make(map[string]*Container)
ids := make(map[string]*Container)
names := make(map[string]*Container)
for n, container := range containers {
idlist = append(idlist, container.ID)
ids[container.ID] = containers[n]
layers[container.LayerID] = containers[n]
for _, name := range container.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
needSave |= containerLocation(container)
}
names[name] = containers[n]
}
Expand All @@ -310,39 +354,69 @@ func (r *containerStore) load(lockedForWriting bool) error {
r.byid = ids
r.bylayer = layers
r.byname = names
if needSave {
if needSave != 0 {
if !lockedForWriting {
// Eventually, the callers should be modified to retry with a write lock, instead.
return errors.New("container store is inconsistent and the current caller does not hold a write lock")
}
return r.Save()
return r.save(needSave)
}
return nil
}

// Save saves the contents of the store to disk. It should be called with
// the lock held, locked for writing.
func (r *containerStore) Save() error {
func (r *containerStore) save(saveLocations containerLocations) error {
r.lockfile.AssertLockedForWriting()
rpath := r.containerspath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
}
jdata, err := json.Marshal(&r.containers)
if err != nil {
return err
}
if err := ioutils.AtomicWriteFile(rpath, jdata, 0600); err != nil {
return err
for locationIndex := 0; locationIndex < numContainerLocationIndex; locationIndex++ {
location := containerLocationFromIndex(locationIndex)
if location&saveLocations == 0 {
continue
}
rpath := r.jsonPath[locationIndex]
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
}
subsetContainers := make([]*Container, 0, len(r.containers))
for _, container := range r.containers {
if containerLocation(container) == location {
subsetContainers = append(subsetContainers, container)
}
}

jdata, err := json.Marshal(&subsetContainers)
if err != nil {
return err
}
var opts *ioutils.AtomicFileWriterOptions
if location == volatileContainerLocation {
opts = &ioutils.AtomicFileWriterOptions{
NoSync: true,
}
}
if err := ioutils.AtomicWriteFileWithOpts(rpath, jdata, 0600, opts); err != nil {
return err
}
}
return r.lockfile.Touch()
}

func newContainerStore(dir string) (rwContainerStore, error) {
func (r *containerStore) saveFor(modifiedContainer *Container) error {
return r.save(containerLocation(modifiedContainer))
}

func newContainerStore(dir string, runDir string, transient bool) (rwContainerStore, error) {
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, err
}
lockfile, err := GetLockfile(filepath.Join(dir, "containers.lock"))
volatileDir := dir
if transient {
if err := os.MkdirAll(runDir, 0700); err != nil {
return nil, err
}
volatileDir = runDir
}
lockfile, err := GetLockfile(filepath.Join(volatileDir, "containers.lock"))
if err != nil {
return nil, err
}
Expand All @@ -353,7 +427,12 @@ func newContainerStore(dir string) (rwContainerStore, error) {
byid: make(map[string]*Container),
bylayer: make(map[string]*Container),
byname: make(map[string]*Container),
jsonPath: [numContainerLocationIndex]string{
filepath.Join(dir, "containers.json"),
filepath.Join(volatileDir, "volatile-containers.json"),
},
}

if err := cstore.startWritingWithReload(false); err != nil {
return nil, err
}
Expand Down Expand Up @@ -385,7 +464,7 @@ func (r *containerStore) ClearFlag(id string, flag string) error {
return ErrContainerUnknown
}
delete(container.Flags, flag)
return r.Save()
return r.saveFor(container)
}

func (r *containerStore) SetFlag(id string, flag string, value interface{}) error {
Expand All @@ -397,7 +476,7 @@ func (r *containerStore) SetFlag(id string, flag string, value interface{}) erro
container.Flags = make(map[string]interface{})
}
container.Flags[flag] = value
return r.Save()
return r.saveFor(container)
}

func (r *containerStore) Create(id string, names []string, image, layer, metadata string, options *ContainerOptions) (container *Container, err error) {
Expand Down Expand Up @@ -443,6 +522,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
Flags: copyStringInterfaceMap(options.Flags),
UIDMap: copyIDMap(options.UIDMap),
GIDMap: copyIDMap(options.GIDMap),
volatileStore: options.Volatile,
}
r.containers = append(r.containers, container)
r.byid[id] = container
Expand All @@ -453,7 +533,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
for _, name := range names {
r.byname[name] = container
}
err = r.Save()
err = r.saveFor(container)
container = copyContainer(container)
return container, err
}
Expand All @@ -468,7 +548,7 @@ func (r *containerStore) Metadata(id string) (string, error) {
func (r *containerStore) SetMetadata(id, metadata string) error {
if container, ok := r.lookup(id); ok {
container.Metadata = metadata
return r.Save()
return r.saveFor(container)
}
return ErrContainerUnknown
}
Expand Down Expand Up @@ -497,7 +577,7 @@ func (r *containerStore) updateNames(id string, names []string, op updateNameOpe
r.byname[name] = container
}
container.Names = names
return r.Save()
return r.saveFor(container)
}

func (r *containerStore) Delete(id string) error {
Expand Down Expand Up @@ -529,7 +609,7 @@ func (r *containerStore) Delete(id string) error {
r.containers = append(r.containers[:toDeleteIndex], r.containers[toDeleteIndex+1:]...)
}
}
if err := r.Save(); err != nil {
if err := r.saveFor(container); err != nil {
return err
}
if err := os.RemoveAll(r.datadir(id)); err != nil {
Expand Down Expand Up @@ -676,7 +756,7 @@ func (r *containerStore) SetBigData(id, key string, data []byte) error {
save = true
}
if save {
err = r.Save()
err = r.saveFor(c)
}
}
return err
Expand Down
15 changes: 11 additions & 4 deletions store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,14 +801,16 @@ func (s *store) load() error {
if err := os.MkdirAll(gcpath, 0700); err != nil {
return err
}
rcs, err := newContainerStore(gcpath)
if err != nil {
return err
}
rcpath := filepath.Join(s.runRoot, driverPrefix+"containers")
if err := os.MkdirAll(rcpath, 0700); err != nil {
return err
}

rcs, err := newContainerStore(gcpath, rcpath, s.transientStore)
if err != nil {
return err
}

s.containerStore = rcs

for _, store := range driver.AdditionalImageStores() {
Expand Down Expand Up @@ -1566,6 +1568,11 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat
}
layer = clayer.ID

// Normally only `--rm` containers are volatile, but in transient store mode all containers are volatile
if s.transientStore {
options.Volatile = true
}

var container *Container
err = s.writeToContainerStore(func(rcstore rwContainerStore) error {
options.IDMappingOptions = types.IDMappingOptions{
Expand Down

0 comments on commit 9beea58

Please sign in to comment.