diff --git a/cmds/ocm/commands/ocmcmds/common/inputs/types/directory/spec.go b/cmds/ocm/commands/ocmcmds/common/inputs/types/directory/spec.go index a3d0f507f..4a96a37ac 100644 --- a/cmds/ocm/commands/ocmcmds/common/inputs/types/directory/spec.go +++ b/cmds/ocm/commands/ocmcmds/common/inputs/types/directory/spec.go @@ -26,6 +26,7 @@ import ( "github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/inputs/cpi" "github.com/open-component-model/ocm/pkg/common/accessio" "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/utils/tarutils" ) type Spec struct { @@ -80,7 +81,7 @@ func (s *Spec) GetBlob(ctx clictx.Context, inputFilePath string) (accessio.Tempo return nil, "", fmt.Errorf("resource type is dir but a file was provided") } - opts := TarFileSystemOptions{ + opts := tarutils.TarFileSystemOptions{ IncludeFiles: s.IncludeFiles, ExcludeFiles: s.ExcludeFiles, PreserveDir: s.PreserveDir != nil && *s.PreserveDir, @@ -96,7 +97,7 @@ func (s *Spec) GetBlob(ctx clictx.Context, inputFilePath string) (accessio.Tempo if s.Compress() { s.SetMediaTypeIfNotDefined(mime.MIME_GZIP) gw := gzip.NewWriter(temp.Writer()) - if err := TarFileSystem(fs, inputPath, gw, opts); err != nil { + if err := tarutils.TarFileSystem(fs, inputPath, gw, opts); err != nil { return nil, "", fmt.Errorf("unable to tar input artifact: %w", err) } if err := gw.Close(); err != nil { @@ -104,7 +105,7 @@ func (s *Spec) GetBlob(ctx clictx.Context, inputFilePath string) (accessio.Tempo } } else { s.SetMediaTypeIfNotDefined(mime.MIME_TAR) - if err := TarFileSystem(fs, inputPath, temp.Writer(), opts); err != nil { + if err := tarutils.TarFileSystem(fs, inputPath, temp.Writer(), opts); err != nil { return nil, "", fmt.Errorf("unable to tar input artifact: %w", err) } } diff --git a/pkg/common/accessio/reserttablereader.go b/pkg/common/accessio/reserttablereader.go index 0df49cf42..e5337fd61 100644 --- a/pkg/common/accessio/reserttablereader.go +++ b/pkg/common/accessio/reserttablereader.go @@ -19,6 +19,8 @@ import ( "io" "io/ioutil" "os" + + "github.com/open-component-model/ocm/pkg/errors" ) type ResettableReader struct { @@ -159,10 +161,5 @@ func (b *fileBuffer) Len() int { } func (b *fileBuffer) Close() error { - err := b.file.Close() - err2 := os.Remove(b.path) - if err2 != nil { - return err2 - } - return err + return errors.ErrListf("closing file buffer").Add(b.file.Close(), os.Remove(b.path)).Result() } diff --git a/pkg/contexts/oci/repositories/artefactset/repository.go b/pkg/contexts/oci/repositories/artefactset/repository.go index ede37c7c1..a590736c7 100644 --- a/pkg/contexts/oci/repositories/artefactset/repository.go +++ b/pkg/contexts/oci/repositories/artefactset/repository.go @@ -97,10 +97,16 @@ func (r Repository) Close() error { return nil } +// NamespaceLister handles the namespaces provided by an artefact set. +// This is always single anonymous namespace, which by ddefinition +// is the empty string. type NamespaceLister struct{} var anonymous cpi.NamespaceLister = &NamespaceLister{} +// NumNamespaces returns the number of namespaces with a given prefix +// for an artefact set. This is either one (the anonymous namespace) if +// the prefix is empty (all namespaces) or zero if a prefix is given. func (n *NamespaceLister) NumNamespaces(prefix string) (int, error) { if prefix == "" { return 1, nil @@ -108,6 +114,9 @@ func (n *NamespaceLister) NumNamespaces(prefix string) (int, error) { return 0, nil } +// GetNamespaces returns namespaces with a given prefix. +// This is the anonymous namespace ("") for an empty prefix +// or no namespace at all if a prefix is given. func (n *NamespaceLister) GetNamespaces(prefix string, closure bool) ([]string, error) { if prefix == "" { return []string{""}, nil diff --git a/pkg/contexts/oci/repositories/docker/namespace.go b/pkg/contexts/oci/repositories/docker/namespace.go index b44f48158..92391cbb4 100644 --- a/pkg/contexts/oci/repositories/docker/namespace.go +++ b/pkg/contexts/oci/repositories/docker/namespace.go @@ -137,7 +137,7 @@ func (n *NamespaceContainer) GetArtefact(vers string) (cpi.ArtefactAccess, error if err != nil { return nil, err } - src, err := ref.NewImageSource(dummyContext, nil) + src, err := ref.NewImageSource(dummyContext, n.repo.sysctx) if err != nil { return nil, err } diff --git a/pkg/utils/tarutils/utils.go b/pkg/utils/tarutils/utils.go new file mode 100644 index 000000000..ca1b97f30 --- /dev/null +++ b/pkg/utils/tarutils/utils.go @@ -0,0 +1,163 @@ +// Copyright 2022 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarutils + +import ( + "archive/tar" + "fmt" + "io" + "os" + pathutil "path" + "path/filepath" + "strings" + + "github.com/mandelsoft/vfs/pkg/vfs" +) + +// TarFileSystemOptions describes additional options for tarring a filesystem. +type TarFileSystemOptions struct { + IncludeFiles []string + ExcludeFiles []string + // PreserveDir defines that the directory specified in the Path field should be included in the blob. + // Only supported for Type dir. + PreserveDir bool + FollowSymlinks bool + + root string +} + +// Included determines whether a file should be included. +func (opts *TarFileSystemOptions) Included(path string) (bool, error) { + // if a root path is given remove it rom the path to be checked + if len(opts.root) != 0 { + path = strings.TrimPrefix(path, opts.root) + } + // first check if a exclude regex matches + for _, ex := range opts.ExcludeFiles { + match, err := filepath.Match(ex, path) + if err != nil { + return false, fmt.Errorf("malformed filepath syntax %q", ex) + } + if match { + return false, nil + } + } + + // if no includes are defined, include all files + if len(opts.IncludeFiles) == 0 { + return true, nil + } + // otherwise check if the file should be included + for _, in := range opts.IncludeFiles { + match, err := filepath.Match(in, path) + if err != nil { + return false, fmt.Errorf("malformed filepath syntax %q", in) + } + if match { + return true, nil + } + } + return false, nil +} + +// TarFileSystem creates a tar archive from a filesystem. +func TarFileSystem(fs vfs.FileSystem, root string, writer io.Writer, opts TarFileSystemOptions) error { + tw := tar.NewWriter(writer) + if opts.PreserveDir { + opts.root = pathutil.Base(root) + } + if err := addFileToTar(fs, tw, opts.root, root, opts); err != nil { + return err + } + return tw.Close() +} + +func addFileToTar(fs vfs.FileSystem, tw *tar.Writer, path string, realPath string, opts TarFileSystemOptions) error { + if len(path) != 0 { // do not check the root + include, err := opts.Included(path) + if err != nil { + return err + } + if !include { + return nil + } + } + info, err := fs.Lstat(realPath) + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = path + + switch { + case info.IsDir(): + // do not write root header + if len(path) != 0 { + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("unable to write header for %q: %w", path, err) + } + } + err := vfs.Walk(fs, realPath, func(subFilePath string, info os.FileInfo, err error) error { + if subFilePath == realPath { + return nil + } + if err != nil { + return err + } + relPath, err := filepath.Rel(realPath, subFilePath) + if err != nil { + return fmt.Errorf("unable to calculate relative path for %s: %w", subFilePath, err) + } + err = addFileToTar(fs, tw, pathutil.Join(path, relPath), subFilePath, opts) + if err == nil && info.IsDir() { + err = vfs.SkipDir + } + return err + }) + return err + case info.Mode().IsRegular(): + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("unable to write header for %q: %w", path, err) + } + file, err := fs.OpenFile(realPath, os.O_RDONLY, os.ModePerm) + if err != nil { + return fmt.Errorf("unable to open file %q: %w", path, err) + } + if _, err := io.Copy(tw, file); err != nil { + _ = file.Close() + return fmt.Errorf("unable to add file to tar %q: %w", path, err) + } + if err := file.Close(); err != nil { + return fmt.Errorf("unable to close file %q: %w", path, err) + } + return nil + case header.Typeflag == tar.TypeSymlink: + if !opts.FollowSymlinks { + //log.Info(fmt.Sprintf("symlink found in %q but symlinks are not followed", path)) + return nil + } + realPath, err := vfs.EvalSymlinks(fs, realPath) + if err != nil { + return fmt.Errorf("unable to follow symlink %s: %w", realPath, err) + } + return addFileToTar(fs, tw, path, realPath, opts) + default: + return fmt.Errorf("unsupported file type %s in %s", info.Mode().String(), path) + } +}