diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index c849027d4..22fd823c7 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.15-alpine +FROM golang:1.16-alpine # Add any build or testing essential system packages RUN apk add --no-cache build-base git pkgconf diff --git a/Dockerfile b/Dockerfile index a8f9704f8..0af148a92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Docker buildkit multi-arch build requires golang alpine -FROM golang:1.15-alpine as builder +FROM golang:1.16-alpine as builder RUN apk add gcc pkgconfig libc-dev RUN apk add --no-cache musl~=1.2 libgit2-dev~=1.1 diff --git a/api/go.mod b/api/go.mod index 6ff883213..f0c2f08c7 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,6 +1,6 @@ module github.com/fluxcd/source-controller/api -go 1.15 +go 1.16 require ( github.com/fluxcd/pkg/apis/meta v0.8.0 diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 78ff38933..3b5fb4a95 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -49,6 +49,7 @@ import ( "github.com/fluxcd/pkg/runtime/predicates" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" + "github.com/fluxcd/source-controller/pkg/sourceignore" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets,verbs=get;list;watch;create;update;patch;delete @@ -202,6 +203,25 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } + // Look for file with ignore rules first + // NB: S3 has flat filepath keys making it impossible to look + // for files in "subdirectories" without building up a tree first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { + if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } + matcher := sourceignore.NewMatcher(ps) + // download bucket content for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{ Recursive: true, @@ -212,7 +232,11 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } - if strings.HasSuffix(object.Key, "/") { + if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { + continue + } + + if matcher.Match([]string{object.Key}, false) { continue } @@ -255,7 +279,7 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket defer unlock() // archive artifact and check integrity - if err := r.Storage.Archive(&artifact, tempDir, bucket.Spec.Ignore); err != nil { + if err := r.Storage.Archive(&artifact, tempDir, nil); err != nil { err = fmt.Errorf("storage archive error: %w", err) return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err } diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 00986aee1..db3bd54ef 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "time" "github.com/go-logr/logr" @@ -45,6 +46,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/strategy" + "github.com/fluxcd/source-controller/pkg/sourceignore" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete @@ -270,7 +272,15 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour defer unlock() // archive artifact and check integrity - if err := r.Storage.Archive(&artifact, tmpGit, repository.Spec.Ignore); err != nil { + ps, err := sourceignore.LoadIgnorePatterns(tmpGit, nil) + if err != nil { + err = fmt.Errorf(".sourceignore error: %w", err) + return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + } + if repository.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*repository.Spec.Ignore), nil)...) + } + if err := r.Storage.Archive(&artifact, tmpGit, SourceIgnoreFilter(ps, nil)); err != nil { err = fmt.Errorf("storage archive error: %w", err) return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err } diff --git a/controllers/storage.go b/controllers/storage.go index 62824f4ec..206f755f7 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -18,8 +18,6 @@ package controllers import ( "archive/tar" - "bufio" - "bytes" "compress/gzip" "crypto/sha1" "fmt" @@ -39,14 +37,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/internal/fs" -) - -const ( - excludeFile = ".sourceignore" - excludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" - excludeExt = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip" - excludeCI = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml" - excludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml" + "github.com/fluxcd/source-controller/pkg/sourceignore" ) // Storage manages artifacts @@ -151,19 +142,35 @@ func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool { return fi.Mode().IsRegular() } -// Archive atomically archives the given directory as a tarball to the given v1beta1.Artifact -// path, excluding any VCS specific files and directories, or any of the excludes defined in -// the excludeFiles. If successful, it sets the checksum and last update time on the artifact. -func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *string) (err error) { - if f, err := os.Stat(dir); os.IsNotExist(err) || !f.IsDir() { - return fmt.Errorf("invalid dir path: %s", dir) +// ArchiveFileFilter must return true if a file should not be included +// in the archive after inspecting the given path and/or os.FileInfo. +type ArchiveFileFilter func(p string, fi os.FileInfo) bool + +// SourceIgnoreFilter returns an ArchiveFileFilter that filters out +// files matching sourceignore.VCSPatterns and any of the provided +// patterns. If an empty gitignore.Pattern slice is given, the matcher +// is set to sourceignore.NewDefaultMatcher. +func SourceIgnoreFilter(ps []gitignore.Pattern, domain []string) ArchiveFileFilter { + matcher := sourceignore.NewDefaultMatcher(ps, domain) + if len(ps) > 0 { + ps = append(sourceignore.VCSPatterns(domain), ps...) + matcher = sourceignore.NewMatcher(ps) + } + return func(p string, fi os.FileInfo) bool { + // The directory is always false as the archiver does already skip + // directories. + return matcher.Match(strings.Split(p, string(filepath.Separator)), false) } +} - ps, err := loadExcludePatterns(dir, ignore) - if err != nil { - return err +// Archive atomically archives the given directory as a tarball to the +// given v1beta1.Artifact path, excluding directories and any +// ArchiveFileFilter matches. If successful, it sets the checksum and +// last update time on the artifact. +func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter ArchiveFileFilter) (err error) { + if f, err := os.Stat(dir); os.IsNotExist(err) || !f.IsDir() { + return fmt.Errorf("invalid dir path: %s", dir) } - matcher := gitignore.NewMatcher(ps) localPath := s.LocalPath(*artifact) tf, err := ioutil.TempFile(filepath.Split(localPath)) @@ -182,43 +189,7 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *strin gw := gzip.NewWriter(mw) tw := tar.NewWriter(gw) - if err := writeToArchiveExcludeMatches(dir, matcher, tw); err != nil { - tw.Close() - gw.Close() - tf.Close() - return err - } - - if err := tw.Close(); err != nil { - gw.Close() - tf.Close() - return err - } - if err := gw.Close(); err != nil { - tf.Close() - return err - } - if err := tf.Close(); err != nil { - return err - } - - if err := os.Chmod(tmpName, 0644); err != nil { - return err - } - - if err := fs.RenameWithFallback(tmpName, localPath); err != nil { - return err - } - - artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) - artifact.LastUpdateTime = metav1.Now() - return nil -} - -// writeToArchiveExcludeMatches walks over the given dir and writes any regular file that does -// not match the given gitignore.Matcher. -func writeToArchiveExcludeMatches(dir string, matcher gitignore.Matcher, writer *tar.Writer) error { - fn := func(p string, fi os.FileInfo, err error) error { + if err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error { if err != nil { return err } @@ -228,8 +199,8 @@ func writeToArchiveExcludeMatches(dir string, matcher gitignore.Matcher, writer return nil } - // Ignore excluded extensions and files - if matcher.Match(strings.Split(p, "/"), false) { + // Skip filtered files + if filter != nil && filter(p, fi) { return nil } @@ -249,7 +220,7 @@ func writeToArchiveExcludeMatches(dir string, matcher gitignore.Matcher, writer } header.Name = relFilePath - if err := writer.WriteHeader(header); err != nil { + if err := tw.WriteHeader(header); err != nil { return err } @@ -258,13 +229,42 @@ func writeToArchiveExcludeMatches(dir string, matcher gitignore.Matcher, writer f.Close() return err } - if _, err := io.Copy(writer, f); err != nil { + if _, err := io.Copy(tw, f); err != nil { f.Close() return err } return f.Close() + }); err != nil { + tw.Close() + gw.Close() + tf.Close() + return err + } + + if err := tw.Close(); err != nil { + gw.Close() + tf.Close() + return err + } + if err := gw.Close(); err != nil { + tf.Close() + return err + } + if err := tf.Close(); err != nil { + return err } - return filepath.Walk(dir, fn) + + if err := os.Chmod(tmpName, 0644); err != nil { + return err + } + + if err := fs.RenameWithFallback(tmpName, localPath); err != nil { + return err + } + + artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) + artifact.LastUpdateTime = metav1.Now() + return nil } // AtomicWriteFile atomically writes the io.Reader contents to the v1beta1.Artifact path. @@ -400,51 +400,6 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string { return filepath.Join(s.BasePath, artifact.Path) } -// getPatterns collects ignore patterns from the given reader and returns them -// as a gitignore.Pattern slice. -func getPatterns(reader io.Reader, path []string) []gitignore.Pattern { - var ps []gitignore.Pattern - scanner := bufio.NewScanner(reader) - - for scanner.Scan() { - s := scanner.Text() - if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { - ps = append(ps, gitignore.ParsePattern(s, path)) - } - } - - return ps -} - -// loadExcludePatterns loads the excluded patterns from sourceignore or other -// sources. -func loadExcludePatterns(dir string, ignore *string) ([]gitignore.Pattern, error) { - path := strings.Split(dir, "/") - - var ps []gitignore.Pattern - for _, p := range strings.Split(excludeVCS, ",") { - ps = append(ps, gitignore.ParsePattern(p, path)) - } - - if ignore == nil { - all := strings.Join([]string{excludeExt, excludeCI, excludeExtra}, ",") - for _, p := range strings.Split(all, ",") { - ps = append(ps, gitignore.ParsePattern(p, path)) - } - - if f, err := os.Open(filepath.Join(dir, excludeFile)); err == nil { - defer f.Close() - ps = append(ps, getPatterns(f, path)...) - } else if !os.IsNotExist(err) { - return nil, err - } - } else { - ps = append(ps, getPatterns(bytes.NewBufferString(*ignore), path)...) - } - - return ps, nil -} - // newHash returns a new SHA1 hash. func newHash() hash.Hash { return sha1.New() diff --git a/controllers/storage_test.go b/controllers/storage_test.go index 3271c5799..a79df6a14 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2020, 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers import ( @@ -7,28 +23,16 @@ import ( "io" "io/ioutil" "os" - "os/exec" "path" "path/filepath" "testing" "time" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" ) -type ignoreMap map[string]bool - -var remoteRepository = "https://github.com/fluxcd/source-controller" - -func init() { - // if this remote repo ever gets in your way, this is an escape; just set - // this to the url you want to clone. Be the source you want to be. - s := os.Getenv("REMOTE_REPOSITORY") - if s != "" { - remoteRepository = s - } -} - func createStoragePath() (string, error) { return ioutil.TempDir("", "") } @@ -67,16 +71,16 @@ func TestStorageConstructor(t *testing.T) { // walks a tar.gz and looks for paths with the basename. It does not match // symlinks properly at this time because that's painful. -func walkTar(tarFile string, match string) (bool, error) { +func walkTar(tarFile string, match string) (int64, bool, error) { f, err := os.Open(tarFile) if err != nil { - return false, fmt.Errorf("could not open file: %w", err) + return 0, false, fmt.Errorf("could not open file: %w", err) } defer f.Close() gzr, err := gzip.NewReader(f) if err != nil { - return false, fmt.Errorf("could not unzip file: %w", err) + return 0, false, fmt.Errorf("could not unzip file: %w", err) } defer gzr.Close() @@ -86,100 +90,23 @@ func walkTar(tarFile string, match string) (bool, error) { if err == io.EOF { break } else if err != nil { - return false, fmt.Errorf("Corrupt tarball reading header: %w", err) + return 0, false, fmt.Errorf("corrupt tarball reading header: %w", err) } switch header.Typeflag { case tar.TypeDir, tar.TypeReg: - if filepath.Base(header.Name) == match { - return true, nil + if header.Name == match { + return header.Size, true, nil } default: // skip } } - return false, nil -} - -func testPatterns(t *testing.T, storage *Storage, artifact sourcev1.Artifact, table ignoreMap) { - for name, expected := range table { - res, err := walkTar(storage.LocalPath(artifact), name) - if err != nil { - t.Fatalf("while reading tarball: %v", err) - } - - if res != expected { - if expected { - t.Fatalf("Could not find repository file matching %q in tarball for repo %q", name, remoteRepository) - } else { - t.Fatalf("Repository contained ignored file %q in tarball for repo %q", name, remoteRepository) - } - } - } -} - -func createArchive(t *testing.T, storage *Storage, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact { - gitDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("could not create temporary directory: %v", err) - } - t.Cleanup(func() { os.RemoveAll(gitDir) }) - - if err := exec.Command("git", "clone", remoteRepository, gitDir).Run(); err != nil { - t.Fatalf("Could not clone remote repository: %v", err) - } - - // inject files.. just empty files - for _, name := range filenames { - f, err := os.Create(filepath.Join(gitDir, name)) - if err != nil { - t.Fatalf("Could not inject filename %q: %v", name, err) - } - f.Close() - } - - // inject sourceignore if not empty - if sourceIgnore != "" { - si, err := os.Create(filepath.Join(gitDir, ".sourceignore")) - if err != nil { - t.Fatalf("Could not create .sourceignore: %v", err) - } - - if _, err := io.WriteString(si, sourceIgnore); err != nil { - t.Fatalf("Could not write to .sourceignore: %v", err) - } - - si.Close() - } - artifact := sourcev1.Artifact{ - Path: filepath.Join(randStringRunes(10), randStringRunes(10), randStringRunes(10)+".tar.gz"), - } - if err := storage.MkdirAll(artifact); err != nil { - t.Fatalf("artifact directory creation failed: %v", err) - } - - if err := storage.Archive(&artifact, gitDir, spec.Ignore); err != nil { - t.Fatalf("archiving failed: %v", err) - } - - if !storage.ArtifactExist(artifact) { - t.Fatalf("artifact was created but does not exist: %+v", artifact) - } - - return artifact + return 0, false, nil } -func stringPtr(s string) *string { - return &s -} - -func TestArchiveBasic(t *testing.T) { - table := ignoreMap{ - "README.md": true, - ".gitignore": false, - } - +func TestStorage_Archive(t *testing.T) { dir, err := createStoragePath() if err != nil { t.Fatal(err) @@ -188,75 +115,131 @@ func TestArchiveBasic(t *testing.T) { storage, err := NewStorage(dir, "hostname", time.Minute) if err != nil { - t.Fatalf("Error while bootstrapping storage: %v", err) + t.Fatalf("error while bootstrapping storage: %v", err) } - testPatterns(t, storage, createArchive(t, storage, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table) -} - -func TestArchiveIgnore(t *testing.T) { - // this is a list of files that will be created in the repository for each - // subtest. it is manipulated later on. - filenames := []string{ - "foo.tar.gz", - "bar.jpg", - "bar.gif", - "foo.jpeg", - "video.flv", - "video.wmv", - "bar.png", - "foo.zip", - ".drone.yml", - ".flux.yaml", - } - - // this is the table of ignored files and their values. true means that it's - // present in the resulting tarball. - table := ignoreMap{} - for _, item := range filenames { - table[item] = false - } - - dir, err := createStoragePath() - if err != nil { - t.Fatal(err) + createFiles := func(files map[string][]byte) (dir string, err error) { + defer func() { + if err != nil && dir != "" { + os.RemoveAll(dir) + } + }() + dir, err = ioutil.TempDir("", "archive-test-files-") + if err != nil { + return + } + for name, b := range files { + absPath := filepath.Join(dir, name) + if err = os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + return + } + f, err := os.Create(absPath) + if err != nil { + return "", fmt.Errorf("could not create file %q: %w", absPath, err) + } + if n, err := f.Write(b); err != nil { + f.Close() + return "", fmt.Errorf("could not write %d bytes to file %q: %w", n, f.Name(), err) + } + f.Close() + } + return } - t.Cleanup(cleanupStoragePath(dir)) - storage, err := NewStorage(dir, "hostname", time.Minute) - if err != nil { - t.Fatalf("Error while bootstrapping storage: %v", err) + matchFiles := func(t *testing.T, storage *Storage, artifact sourcev1.Artifact, files map[string][]byte) { + for name, b := range files { + mustExist := !(name[0:1] == "!") + if !mustExist { + name = name[1:] + } + s, exist, err := walkTar(storage.LocalPath(artifact), name) + if err != nil { + t.Fatalf("failed reading tarball: %v", err) + } + if bs := int64(len(b)); s != bs { + t.Fatalf("%q size %v != %v", name, s, bs) + } + if exist != mustExist { + if mustExist { + t.Errorf("could not find file %q in tarball", name) + } else { + t.Errorf("tarball contained excluded file %q", name) + } + } + } } - t.Run("automatically ignored files", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{}), table) - }) - - table = ignoreMap{} - for _, item := range filenames { - table[item] = true + tests := []struct { + name string + files map[string][]byte + filter ArchiveFileFilter + want map[string][]byte + wantErr bool + }{ + { + name: "no filter", + files: map[string][]byte{ + ".git/config": nil, + "file.jpg": []byte(`contents`), + "manifest.yaml": nil, + }, + filter: nil, + want: map[string][]byte{ + ".git/config": nil, + "file.jpg": []byte(`contents`), + "manifest.yaml": nil, + }, + }, + { + name: "exclude VCS", + files: map[string][]byte{ + ".git/config": nil, + "manifest.yaml": nil, + }, + filter: SourceIgnoreFilter(nil, nil), + want: map[string][]byte{ + "!.git/config": nil, + "manifest.yaml": nil, + }, + }, + { + name: "custom", + files: map[string][]byte{ + ".git/config": nil, + "custom": nil, + "horse.jpg": nil, + }, + filter: SourceIgnoreFilter([]gitignore.Pattern{ + gitignore.ParsePattern("custom", nil), + }, nil), + want: map[string][]byte{ + "!git/config": nil, + "!custom": nil, + "horse.jpg": nil, + }, + wantErr: false, + }, } - - t.Run("only vcs ignored files", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table) - }) - - filenames = append(filenames, "test.txt") - table["test.txt"] = false - sourceIgnoreFile := "*.txt" - - t.Run("sourceignore injected via CRD", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table) - }) - - table = ignoreMap{} - for _, item := range filenames { - table[item] = false + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := createFiles(tt.files) + if err != nil { + t.Error(err) + return + } + defer os.RemoveAll(dir) + artifact := sourcev1.Artifact{ + Path: filepath.Join(randStringRunes(10), randStringRunes(10), randStringRunes(10)+".tar.gz"), + } + if err := storage.MkdirAll(artifact); err != nil { + t.Fatalf("artifact directory creation failed: %v", err) + } + if err := storage.Archive(&artifact, dir, tt.filter); (err != nil) != tt.wantErr { + t.Errorf("Archive() error = %v, wantErr %v", err, tt.wantErr) + } + matchFiles(t, storage, artifact, tt.want) + }) } - - t.Run("sourceignore injected via filename", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table) - }) } func TestStorageRemoveAllButCurrent(t *testing.T) { diff --git a/go.mod b/go.mod index eac89da88..75a9f325c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/fluxcd/source-controller -go 1.15 +go 1.16 replace github.com/fluxcd/source-controller/api => ./api @@ -28,6 +28,7 @@ require ( github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.5.3 k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 diff --git a/pkg/git/gogit/transport.go b/pkg/git/gogit/transport.go index 2d51c0308..f07e10f5b 100644 --- a/pkg/git/gogit/transport.go +++ b/pkg/git/gogit/transport.go @@ -88,7 +88,8 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { user = git.DefaultPublicKeyAuthUser } - pk, err := ssh.NewPublicKeys(user, identity, "") + password := secret.Data["password"] + pk, err := ssh.NewPublicKeys(user, identity, string(password)) if err != nil { return nil, err } diff --git a/pkg/git/gogit/transport_test.go b/pkg/git/gogit/transport_test.go index 675ce66cb..2213dbcef 100644 --- a/pkg/git/gogit/transport_test.go +++ b/pkg/git/gogit/transport_test.go @@ -43,6 +43,21 @@ v2MYnxRjc9INpi/Dyzz2MMvOnOW+aDuOh/If2AtVCmeJUx1pf4CFk3viQwJBAKyC t824+evjv+NQBlme3AOF6PgxtV4D4wWoJ5Uk/dTejER0j/Hbl6sqPxuiILRRV9qJ Ngkgu4mLjc3RfenEhJECQAx8zjWUE6kHHPGAd9DfiAIQ4bChqnyS0Nwb9+Gd4hSE P0Ah10mHiK/M0o3T8Eanwum0gbQHPnOwqZgsPkwXRqQ= +-----END RSA PRIVATE KEY-----` + + // secretKeyFixture is a randomly generated + // 512bit RSA private key with password foobar. + secretPassphraseFixture = `-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,0B016973B2A761D31E6B388D0F327C35 + +X9GET/qAyZkAJBl/RK+1XX75NxONgdUfZDw7PIYi/g+Efh3Z5zH5kh/dx9lxH5ZG +HGCqPAeMO/ofGDGtDULWW6iqDUFRu5gPgEVSCnnbqoHNU325WHhXdhejVAItwObC +IpL/zYfs2+gDHXct/n9FJ/9D/EGXZihwPqYaK8GQSfZAxz0QjLuh0wU1qpbm3y3N +q+o9FLv3b2Ys/tCJOUsYVQOYLSrZEI77y1ii3nWgQ8lXiTJbBUKzuq4f1YWeO8Ah +RZbdhTa57AF5lUaRtL7Nrm3HJUrK1alBbU7HHyjeW4Q4n/D3fiRDC1Mh2Bi4EOOn +wGctSx4kHsZGhJv5qwKqqPEFPhUzph8D2tm2TABk8HJa5KJFDbGrcfvk2uODAoZr +MbcpIxCfl8oB09bWfY6tDQjyvwSYYo2Phdwm7kT92xc= -----END RSA PRIVATE KEY-----` // knownHostsFixture is known_hosts fixture in the expected @@ -63,6 +78,13 @@ var ( "known_hosts": []byte(knownHostsFixture), }, } + privateKeySecretWithPassphraseFixture = corev1.Secret{ + Data: map[string][]byte{ + "identity": []byte(secretPassphraseFixture), + "known_hosts": []byte(knownHostsFixture), + "password": []byte("foobar"), + }, + } ) func TestAuthSecretStrategyForURL(t *testing.T) { @@ -131,10 +153,13 @@ func TestPublicKeyStrategy_Method(t *testing.T) { wantErr bool }{ {"private key and known_hosts", privateKeySecretFixture, nil, false}, + {"private key with passphrase and known_hosts", privateKeySecretWithPassphraseFixture, nil, false}, {"missing private key", privateKeySecretFixture, func(s *corev1.Secret) { delete(s.Data, "identity") }, true}, {"invalid private key", privateKeySecretFixture, func(s *corev1.Secret) { s.Data["identity"] = []byte(`-----BEGIN RSA PRIVATE KEY-----`) }, true}, {"missing known_hosts", privateKeySecretFixture, func(s *corev1.Secret) { delete(s.Data, "known_hosts") }, true}, {"invalid known_hosts", privateKeySecretFixture, func(s *corev1.Secret) { s.Data["known_hosts"] = []byte(`invalid`) }, true}, + {"missing password", privateKeySecretWithPassphraseFixture, func(s *corev1.Secret) { delete(s.Data, "password") }, true}, + {"wrong password", privateKeySecretWithPassphraseFixture, func(s *corev1.Secret) { s.Data["password"] = []byte("pass") }, true}, {"empty", corev1.Secret{}, nil, true}, } for _, tt := range tests { diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go index f53273567..da3d04e92 100644 --- a/pkg/git/libgit2/transport.go +++ b/pkg/git/libgit2/transport.go @@ -121,7 +121,13 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { // Need to validate private key as it is not // done by git2go when loading the key - _, err = ssh.ParsePrivateKey(identity) + password, ok := secret.Data["password"] + if ok { + _, err = ssh.ParsePrivateKeyWithPassphrase(identity, password) + } else { + _, err = ssh.ParsePrivateKey(identity) + } + if err != nil { return nil, err } @@ -132,7 +138,7 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { } credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), "") + cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), string(password)) if err != nil { return nil, err } diff --git a/pkg/git/libgit2/transport_test.go b/pkg/git/libgit2/transport_test.go index 2a1387c1d..733fa0c96 100644 --- a/pkg/git/libgit2/transport_test.go +++ b/pkg/git/libgit2/transport_test.go @@ -44,6 +44,21 @@ v2MYnxRjc9INpi/Dyzz2MMvOnOW+aDuOh/If2AtVCmeJUx1pf4CFk3viQwJBAKyC t824+evjv+NQBlme3AOF6PgxtV4D4wWoJ5Uk/dTejER0j/Hbl6sqPxuiILRRV9qJ Ngkgu4mLjc3RfenEhJECQAx8zjWUE6kHHPGAd9DfiAIQ4bChqnyS0Nwb9+Gd4hSE P0Ah10mHiK/M0o3T8Eanwum0gbQHPnOwqZgsPkwXRqQ= +-----END RSA PRIVATE KEY-----` + + // secretKeyFixture is a randomly generated + // 512bit RSA private key with password foobar. + secretPassphraseFixture = `-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,0B016973B2A761D31E6B388D0F327C35 + +X9GET/qAyZkAJBl/RK+1XX75NxONgdUfZDw7PIYi/g+Efh3Z5zH5kh/dx9lxH5ZG +HGCqPAeMO/ofGDGtDULWW6iqDUFRu5gPgEVSCnnbqoHNU325WHhXdhejVAItwObC +IpL/zYfs2+gDHXct/n9FJ/9D/EGXZihwPqYaK8GQSfZAxz0QjLuh0wU1qpbm3y3N +q+o9FLv3b2Ys/tCJOUsYVQOYLSrZEI77y1ii3nWgQ8lXiTJbBUKzuq4f1YWeO8Ah +RZbdhTa57AF5lUaRtL7Nrm3HJUrK1alBbU7HHyjeW4Q4n/D3fiRDC1Mh2Bi4EOOn +wGctSx4kHsZGhJv5qwKqqPEFPhUzph8D2tm2TABk8HJa5KJFDbGrcfvk2uODAoZr +MbcpIxCfl8oB09bWfY6tDQjyvwSYYo2Phdwm7kT92xc= -----END RSA PRIVATE KEY-----` // knownHostsFixture is known_hosts fixture in the expected @@ -64,6 +79,13 @@ var ( "known_hosts": []byte(knownHostsFixture), }, } + privateKeySecretWithPassphraseFixture = corev1.Secret{ + Data: map[string][]byte{ + "identity": []byte(secretPassphraseFixture), + "known_hosts": []byte(knownHostsFixture), + "password": []byte("foobar"), + }, + } ) func TestAuthSecretStrategyForURL(t *testing.T) { @@ -126,10 +148,13 @@ func TestPublicKeyStrategy_Method(t *testing.T) { wantErr bool }{ {"private key and known_hosts", privateKeySecretFixture, nil, false}, + {"private key with passphrase and known_hosts", privateKeySecretWithPassphraseFixture, nil, false}, {"missing private key", privateKeySecretFixture, func(s *corev1.Secret) { delete(s.Data, "identity") }, true}, {"invalid private key", privateKeySecretFixture, func(s *corev1.Secret) { s.Data["identity"] = []byte(`-----BEGIN RSA PRIVATE KEY-----`) }, true}, {"missing known_hosts", privateKeySecretFixture, func(s *corev1.Secret) { delete(s.Data, "known_hosts") }, true}, {"invalid known_hosts", privateKeySecretFixture, func(s *corev1.Secret) { s.Data["known_hosts"] = []byte(`invalid`) }, true}, + {"missing password", privateKeySecretWithPassphraseFixture, func(s *corev1.Secret) { delete(s.Data, "password") }, true}, + {"invalid password", privateKeySecretWithPassphraseFixture, func(s *corev1.Secret) { s.Data["password"] = []byte("foo") }, true}, {"empty", corev1.Secret{}, nil, true}, } for _, tt := range tests { diff --git a/pkg/sourceignore/sourceignore.go b/pkg/sourceignore/sourceignore.go new file mode 100644 index 000000000..b4e0bf50f --- /dev/null +++ b/pkg/sourceignore/sourceignore.go @@ -0,0 +1,125 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sourceignore + +import ( + "bufio" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5/plumbing/format/gitignore" +) + +const ( + IgnoreFile = ".sourceignore" + ExcludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" + ExcludeExt = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip" + ExcludeCI = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml" + ExcludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml" +) + +// NewMatcher returns a gitignore.Matcher for the given gitignore.Pattern +// slice. It mainly exists to compliment the API. +func NewMatcher(ps []gitignore.Pattern) gitignore.Matcher { + return gitignore.NewMatcher(ps) +} + +// NewDefaultMatcher returns a gitignore.Matcher with the DefaultPatterns +// as lowest priority patterns. +func NewDefaultMatcher(ps []gitignore.Pattern, domain []string) gitignore.Matcher { + var defaultPs []gitignore.Pattern + defaultPs = append(defaultPs, VCSPatterns(domain)...) + defaultPs = append(defaultPs, DefaultPatterns(domain)...) + ps = append(defaultPs, ps...) + return gitignore.NewMatcher(ps) +} + +// VCSPatterns returns a gitignore.Pattern slice with ExcludeVCS +// patterns. +func VCSPatterns(domain []string) []gitignore.Pattern { + var ps []gitignore.Pattern + for _, p := range strings.Split(ExcludeVCS, ",") { + ps = append(ps, gitignore.ParsePattern(p, domain)) + } + return ps +} + +// DefaultPatterns returns a gitignore.Pattern slice with the default +// ExcludeExt, ExcludeCI, ExcludeExtra patterns. +func DefaultPatterns(domain []string) []gitignore.Pattern { + all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",") + var ps []gitignore.Pattern + for _, p := range strings.Split(all, ",") { + ps = append(ps, gitignore.ParsePattern(p, domain)) + } + return ps +} + +// ReadPatterns collects ignore patterns from the given reader and +// returns them as a gitignore.Pattern slice. +// If a domain is supplied, this is used as the scope of the read +// patterns. +func ReadPatterns(reader io.Reader, domain []string) []gitignore.Pattern { + var ps []gitignore.Pattern + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + s := scanner.Text() + if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { + ps = append(ps, gitignore.ParsePattern(s, domain)) + } + } + return ps +} + +// ReadIgnoreFile attempts to read the file at the given path and +// returns the read patterns. +func ReadIgnoreFile(path string, domain []string) ([]gitignore.Pattern, error) { + var ps []gitignore.Pattern + if f, err := os.Open(path); err == nil { + defer f.Close() + ps = append(ps, ReadPatterns(f, domain)...) + } else if !os.IsNotExist(err) { + return nil, err + } + return ps, nil +} + +// LoadIgnorePatterns recursively loads the the IgnoreFile patterns found +// in the directory. +func LoadIgnorePatterns(dir string, domain []string) ([]gitignore.Pattern, error) { + ps, err := ReadIgnoreFile(filepath.Join(dir, IgnoreFile), domain) + if err != nil { + return nil, err + } + fis, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + for _, fi := range fis { + if fi.IsDir() && fi.Name() != ".git" { + var subps []gitignore.Pattern + subps, err = LoadIgnorePatterns(filepath.Join(dir, fi.Name()), append(domain, fi.Name())) + if len(subps) > 0 { + ps = append(ps, subps...) + } + } + } + return ps, nil +} diff --git a/pkg/sourceignore/sourceignore_test.go b/pkg/sourceignore/sourceignore_test.go new file mode 100644 index 000000000..98a88d7e0 --- /dev/null +++ b/pkg/sourceignore/sourceignore_test.go @@ -0,0 +1,261 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sourceignore + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "gotest.tools/assert" +) + +func TestReadPatterns(t *testing.T) { + tests := []struct { + name string + ignore string + domain []string + matches []string + mismatches []string + }{ + { + name: "simple", + ignore: `ignore-dir/* +!ignore-dir/include +`, + matches: []string{"ignore-dir/file.yaml"}, + mismatches: []string{"file.yaml", "ignore-dir/include"}, + }, + { + name: "with comments", + ignore: `ignore-dir/* +# !ignore-dir/include`, + matches: []string{"ignore-dir/file.yaml", "ignore-dir/include"}, + }, + { + name: "domain scoped", + domain: []string{"domain", "scoped"}, + ignore: "ignore-dir/*", + matches: []string{"domain/scoped/ignore-dir/file.yaml"}, + mismatches: []string{"ignore-dir/file.yaml"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := strings.NewReader(tt.ignore) + ps := ReadPatterns(reader, tt.domain) + matcher := NewMatcher(ps) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestReadIgnoreFile(t *testing.T) { + f, err := ioutil.TempFile("", IgnoreFile) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err = f.Write([]byte(`# .sourceignore +ignore-this.txt`)); err != nil { + t.Fatal(err) + } + f.Close() + + tests := []struct { + name string + path string + domain []string + want []gitignore.Pattern + }{ + { + name: IgnoreFile, + path: f.Name(), + want: []gitignore.Pattern{ + gitignore.ParsePattern("ignore-this.txt", nil), + }, + }, + { + name: "with domain", + path: f.Name(), + domain: strings.Split(filepath.Dir(f.Name()), string(filepath.Separator)), + want: []gitignore.Pattern{ + gitignore.ParsePattern("ignore-this.txt", strings.Split(filepath.Dir(f.Name()), string(filepath.Separator))), + }, + }, + { + name: "non existing", + path: "", + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ReadIgnoreFile(tt.path, tt.domain) + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadIgnoreFile() got = %d, want %#v", got, tt.want) + } + }) + } +} + +func TestVCSPatterns(t *testing.T) { + tests := []struct { + name string + domain []string + patterns []gitignore.Pattern + matches []string + mismatches []string + }{ + { + name: "simple matches", + matches: []string{".git/config", ".gitignore"}, + mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, + }, + { + name: "domain scoped matches", + domain: []string{"directory"}, + matches: []string{"directory/.git/config", "directory/.gitignore"}, + mismatches: []string{"other/.git/config"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := NewDefaultMatcher(tt.patterns, tt.domain) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestDefaultPatterns(t *testing.T) { + tests := []struct { + name string + domain []string + patterns []gitignore.Pattern + matches []string + mismatches []string + }{ + { + name: "simple matches", + matches: []string{"image.jpg", "archive.tar.gz", ".github/workflows/workflow.yaml", "subdir/.flux.yaml", "subdir2/.sops.yaml"}, + mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, + }, + { + name: "domain scoped matches", + domain: []string{"directory"}, + matches: []string{"directory/image.jpg", "directory/archive.tar.gz"}, + mismatches: []string{"other/image.jpg", "other/archive.tar.gz"}, + }, + { + name: "patterns", + patterns: []gitignore.Pattern{gitignore.ParsePattern("!*.jpg", nil)}, + mismatches: []string{"image.jpg"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := NewDefaultMatcher(tt.patterns, tt.domain) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestLoadExcludePatterns(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "sourceignore-load-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + files := map[string]string{ + ".sourceignore": "root.txt", + "d/.gitignore": "ignored", + "z/.sourceignore": "last.txt", + "a/b/.sourceignore": "subdir.txt", + } + for n, c := range files { + if err = os.MkdirAll(filepath.Join(tmpDir, filepath.Dir(n)), 0755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(tmpDir, n), []byte(c), 0644); err != nil { + t.Fatal(err) + } + } + tests := []struct { + name string + dir string + domain []string + want []gitignore.Pattern + }{ + { + name: "traverse loads", + dir: tmpDir, + want: []gitignore.Pattern{ + gitignore.ParsePattern("root.txt", nil), + gitignore.ParsePattern("subdir.txt", []string{"a", "b"}), + gitignore.ParsePattern("last.txt", []string{"z"}), + }, + }, + { + name: "domain", + dir: tmpDir, + domain: strings.Split(tmpDir, string(filepath.Separator)), + want: []gitignore.Pattern{ + gitignore.ParsePattern("root.txt", strings.Split(tmpDir, string(filepath.Separator))), + gitignore.ParsePattern("subdir.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "a", "b")), + gitignore.ParsePattern("last.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "z")), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := LoadIgnorePatterns(tt.dir, tt.domain) + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("LoadIgnorePatterns() got = %#v, want %#v", got, tt.want) + for _, v := range got { + t.Error(v) + } + } + }) + } +}