Add option to provide a folder as an input. Fixes #121 (#122)

.goreleaser.yml

@@ -1,3 +1,4 @@
+version: 2
 builds:
   - env:
       - CGO_ENABLED=0

README.md

@@ -13,7 +13,7 @@
   - [Examples](#examples)
   - [Contributing \& Roadmap](#contributing--roadmap)
 
-`kubectl-slice` is a neat tool that allows you to split a single multi-YAML Kubernetes manifest into multiple subfiles using a naming convention you choose. This is done by parsing the YAML code and allowing you to access any key from the YAML object [using Go Templates](https://pkg.go.dev/text/template).
+`kubectl-slice` is a tool that allows you to split a single multi-YAML Kubernetes manifest (with `--input-file` or `-f`), or a folder containing multiple manifests files (with `--input-folder` or `-d`, optionally with `--recursive`), into multiple subfiles using a naming convention you choose. This is done by parsing the YAML code and allowing you to access any key from the YAML object [using Go Templates](https://pkg.go.dev/text/template).
 
 By default, `kubectl-slice` will split your files into multiple subfiles following this naming convention that you can configure to your liking:
 
@@ -86,6 +86,8 @@ Examples:
   kubectl-slice -f foo.yaml --exclude-name *-svc --stdout
   kubectl-slice -f foo.yaml --include Pod/* --stdout
   kubectl-slice -f foo.yaml --exclude deployment/kube* --stdout
+  kubectl-slice -d ./ --recurse -o ./ --include-kind Pod,Namespace
+  kubectl-slice -d ./ --recurse --stdout --include Pod/*
   kubectl-slice --config config.yaml
 
 Flags:
@@ -96,15 +98,18 @@ Flags:
       --exclude strings        resource name to exclude in the output (format <kind>/<name>, case insensitive, glob supported)
       --exclude-kind strings   resource kind to exclude in the output (singular, case insensitive, glob supported)
       --exclude-name strings   resource name to exclude in the output (singular, case insensitive, glob supported)
+      --extensions strings     the extensions to look for in the input folder (default [.yaml,.yml])
   -h, --help                   help for kubectl-slice
       --include strings        resource name to include in the output (format <kind>/<name>, case insensitive, glob supported)
       --include-kind strings   resource kind to include in the output (singular, case insensitive, glob supported)
       --include-name strings   resource name to include in the output (singular, case insensitive, glob supported)
       --include-triple-dash    if enabled, the typical "---" YAML separator is included at the beginning of resources sliced
-  -f, --input-file string      the input file used to read the initial macro YAML file; if empty or "-", stdin is used
+  -f, --input-file string      the input file used to read the initial macro YAML file; if empty or "-", stdin is used (exclusive with --input-folder)
+  -d, --input-folder string    the input folder used to read the initial macro YAML files (exclusive with --input-file)
   -o, --output-dir string      the output directory used to output the splitted files
       --prune                  if enabled, the output directory will be pruned before writing the files
   -q, --quiet                  if true, no output is written to stdout/err
+  -r, --recurse                if true, the input folder will be read recursively (has no effect unless used with --input-folder)
   -s, --skip-non-k8s           if enabled, any YAMLs that don't contain at least an "apiVersion", "kind" and "metadata.name" will be excluded from the split
       --sort-by-kind           if enabled, resources are sorted by Kind, a la Helm, before saving them to disk
       --stdout                 if enabled, no resource is written to disk and all resources are printed to stdout instead

app.go

@@ -28,6 +28,8 @@ var examples = []string{
 	"kubectl-slice -f foo.yaml --exclude-name *-svc --stdout",
 	"kubectl-slice -f foo.yaml --include Pod/* --stdout",
 	"kubectl-slice -f foo.yaml --exclude deployment/kube* --stdout",
+	"kubectl-slice -d ./ --recurse -o ./ --include-kind Pod,Namespace",
+	"kubectl-slice -d ./ --recurse --stdout --include Pod/*",
 	"kubectl-slice --config config.yaml",
 }
 
@@ -68,7 +70,7 @@ func root() *cobra.Command {
 
 			// If no input file has been provided or it's "-", then
 			// point the app to stdin
-			if opts.InputFile == "" || opts.InputFile == "-" {
+			if (opts.InputFile == "" || opts.InputFile == "-") && opts.InputFolder == "" {
 				opts.InputFile = os.Stdin.Name()
 
 				// Check if we're receiving data from the terminal
@@ -94,7 +96,10 @@ func root() *cobra.Command {
 		},
 	}
 
-	rootCommand.Flags().StringVarP(&opts.InputFile, "input-file", "f", "", "the input file used to read the initial macro YAML file; if empty or \"-\", stdin is used")
+	rootCommand.Flags().StringVarP(&opts.InputFile, "input-file", "f", "", "the input file used to read the initial macro YAML file; if empty or \"-\", stdin is used (exclusive with --input-folder)")
+	rootCommand.Flags().StringVarP(&opts.InputFolder, "input-folder", "d", "", "the input folder used to read the initial macro YAML files (exclusive with --input-file)")
+	rootCommand.Flags().StringSliceVar(&opts.InputFolderExt, "extensions", []string{".yaml", ".yml"}, "the extensions to look for in the input folder")
+	rootCommand.Flags().BoolVarP(&opts.Recurse, "recurse", "r", false, "if true, the input folder will be read recursively (has no effect unless used with --input-folder)")
 	rootCommand.Flags().StringVarP(&opts.OutputDirectory, "output-dir", "o", "", "the output directory used to output the splitted files")
 	rootCommand.Flags().StringVarP(&opts.GoTemplate, "template", "t", slice.DefaultTemplateName, "go template used to generate the file name when creating the resource files in the output directory")
 	rootCommand.Flags().BoolVar(&opts.DryRun, "dry-run", false, "if true, no files are created, but the potentially generated files will be printed as the command output")

docs/configuring-cli.md

@@ -35,6 +35,9 @@ The following is an example of a configuration file with the types defined:
 
 ```yaml
 input_file: string
+input_dir: string
+extensions: [string]
+recurse: boolean
 output_dir: string
 template: string
 dry_run: boolean

docs/examples.md

@@ -2,6 +2,7 @@
 
 - [Examples](#examples)
   - [Slicing the Tekton manifest](#slicing-the-tekton-manifest)
+  - [Finding all Kubernetes resources of a given kind in multiple YAML files in a folder](#finding-all-kubernetes-resources-of-a-given-kind-in-multiple-yaml-files-in-a-folder)
 
 The following examples demonstrate the capabilities of `kubectl-slice`.
 
@@ -123,3 +124,106 @@ configmap
 
 0 directories, 9 files
 ```
+
+## Finding all Kubernetes resources of a given kind in multiple YAML files in a folder
+
+Imagine you have a folder with several YAML files. Each file may contain one to many Kubernetes resources. You want to find all resources of a given kind, for example, all `Secret` resources.
+
+As an example, let's clone the ArgoCD repository, which has a nifty `manifests/` folder. Say we want to find all the secrets-type files from the `base` folder in `manifests/base/`, looking at all the YAML files in that folder, we have:
+
+```bash
+$ find ./manifests/base -type f -name "*.yaml"
+./manifests/base/application-controller-roles/argocd-application-controller-role.yaml
+./manifests/base/application-controller-roles/kustomization.yaml
+./manifests/base/application-controller-roles/argocd-application-controller-rolebinding.yaml
+./manifests/base/application-controller-roles/argocd-application-controller-sa.yaml
+./manifests/base/application-controller/kustomization.yaml
+./manifests/base/application-controller/argocd-application-controller-statefulset.yaml
+./manifests/base/application-controller/argocd-metrics.yaml
+./manifests/base/application-controller/argocd-application-controller-network-policy.yaml
+./manifests/base/application-controller-deployment/argocd-application-controller-deployment.yaml
+./manifests/base/application-controller-deployment/argocd-application-controller-service.yaml
+./manifests/base/application-controller-deployment/kustomization.yaml
+./manifests/base/application-controller-deployment/argocd-application-controller-statefulset.yaml
+./manifests/base/config/argocd-cm.yaml
+./manifests/base/config/kustomization.yaml
+./manifests/base/config/argocd-cmd-params-cm.yaml
+./manifests/base/config/argocd-gpg-keys-cm.yaml
+./manifests/base/config/argocd-tls-certs-cm.yaml
+./manifests/base/config/argocd-ssh-known-hosts-cm.yaml
+./manifests/base/config/argocd-rbac-cm.yaml
+./manifests/base/config/argocd-secret.yaml
+./manifests/base/redis/argocd-redis-service.yaml
+./manifests/base/redis/kustomization.yaml
+./manifests/base/redis/argocd-redis-role.yaml
+./manifests/base/redis/argocd-redis-deployment.yaml
+./manifests/base/redis/argocd-redis-rolebinding.yaml
+./manifests/base/redis/argocd-redis-sa.yaml
+./manifests/base/redis/argocd-redis-network-policy.yaml
+./manifests/base/notification/argocd-notifications-controller-network-policy.yaml
+./manifests/base/notification/kustomization.yaml
+./manifests/base/notification/argocd-notifications-controller-rolebinding.yaml
+./manifests/base/notification/argocd-notifications-controller-sa.yaml
+./manifests/base/notification/argocd-notifications-controller-metrics-service.yaml
+./manifests/base/notification/argocd-notifications-cm.yaml
+./manifests/base/notification/argocd-notifications-controller-deployment.yaml
+./manifests/base/notification/argocd-notifications-secret.yaml
+./manifests/base/notification/argocd-notifications-controller-role.yaml
+./manifests/base/repo-server/argocd-repo-server-network-policy.yaml
+./manifests/base/repo-server/argocd-repo-server-service.yaml
+./manifests/base/repo-server/argocd-repo-server-deployment.yaml
+./manifests/base/repo-server/kustomization.yaml
+./manifests/base/repo-server/argocd-repo-server-sa.yaml
+./manifests/base/kustomization.yaml
+./manifests/base/server/argocd-server-rolebinding.yaml
+./manifests/base/server/kustomization.yaml
+./manifests/base/server/argocd-server-network-policy.yaml
+./manifests/base/server/argocd-server-role.yaml
+./manifests/base/server/argocd-server-deployment.yaml
+./manifests/base/server/argocd-server-metrics.yaml
+./manifests/base/server/argocd-server-sa.yaml
+./manifests/base/server/argocd-server-service.yaml
+./manifests/base/dex/argocd-dex-server-rolebinding.yaml
+./manifests/base/dex/argocd-dex-server-service.yaml
+./manifests/base/dex/kustomization.yaml
+./manifests/base/dex/argocd-dex-server-network-policy.yaml
+./manifests/base/dex/argocd-dex-server-sa.yaml
+./manifests/base/dex/argocd-dex-server-deployment.yaml
+./manifests/base/dex/argocd-dex-server-role.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-role.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-rolebinding.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-deployment.yaml
+./manifests/base/applicationset-controller/kustomization.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-service.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-network-policy.yaml
+./manifests/base/applicationset-controller/argocd-applicationset-controller-sa.yaml
+```
+
+It would be time-consuming to try to process them manually.
+
+Let's ask `kubectl-slice` to get us only the `Secrets`. Since there are some `kustomize` files in there, we'll exclude those, which fit the criteria for `--skip-non-k8s` since they don't have a `metadata.name` field. Let's print those to `stdout` as well:
+
+```bash
+$ kubectl-slice -d ./manifests/base --recurse --include-kind Secret --skip-non-k8s --stdout
+# File: secret-argocd-secret.yaml (162 bytes)
+apiVersion: v1
+kind: Secret
+metadata:
+  name: argocd-secret
+  labels:
+    app.kubernetes.io/name: argocd-secret
+    app.kubernetes.io/part-of: argocd
+type: Opaque
+---
+# File: secret-argocd-notifications-secret.yaml (252 bytes)
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-secret
+type: Opaque
+2 files parsed to stdout.
+```

slice/split.go

@@ -63,15 +63,18 @@ type Options struct {
 	Stdout io.Writer
 	Stderr io.Writer
 
-	InputFile         string // the name of the input file to be read
-	OutputDirectory   string // the path to the directory where the files will be stored
-	PruneOutputDir    bool   // if true, the output directory will be pruned before writing the files
-	OutputToStdout    bool   // if true, the output will be written to stdout instead of a file
-	GoTemplate        string // the go template code to render the file names
-	DryRun            bool   // if true, no files are created
-	DebugMode         bool   // enables debug mode
-	Quiet             bool   // disables all writing to stdout/stderr
-	IncludeTripleDash bool   // include the "---" separator on resources sliced
+	InputFile         string   // the name of the input file to be read
+	InputFolder       string   // the name of the input folder to be read
+	InputFolderExt    []string // the extensions of the files to be read
+	Recurse           bool     // if true, the input folder will be read recursively
+	OutputDirectory   string   // the path to the directory where the files will be stored
+	PruneOutputDir    bool     // if true, the output directory will be pruned before writing the files
+	OutputToStdout    bool     // if true, the output will be written to stdout instead of a file
+	GoTemplate        string   // the go template code to render the file names
+	DryRun            bool     // if true, no files are created
+	DebugMode         bool     // enables debug mode
+	Quiet             bool     // disables all writing to stdout/stderr
+	IncludeTripleDash bool     // include the "---" separator on resources sliced
 
 	IncludedKinds    []string
 	ExcludedKinds    []string

slice/utils.go

@@ -5,8 +5,69 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strings"
 )
 
+func inarray[T comparable](needle T, haystack []T) bool {
+	for _, v := range haystack {
+		if v == needle {
+			return true
+		}
+	}
+
+	return false
+}
+
+// loadfolder reads the folder contents recursively for `.yaml` and `.yml` files
+// and returns a buffer with the contents of all files found; returns the buffer
+// with all the files separated by `---` and the number of files found
+func loadfolder(extensions []string, folderPath string, recurse bool) (*bytes.Buffer, int, error) {
+	var buffer bytes.Buffer
+	var count int
+
+	err := filepath.Walk(folderPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			if path != folderPath && !recurse {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		ext := strings.ToLower(filepath.Ext(path))
+		if inarray(ext, extensions) {
+			count++
+
+			data, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			if buffer.Len() > 0 {
+				buffer.WriteString("\n---\n")
+			}
+
+			buffer.Write(data)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, 0, err
+	}
+
+	if buffer.Len() == 0 {
+		return nil, 0, fmt.Errorf("no files found in %q with extensions: %s", folderPath, strings.Join(extensions, ", "))
+	}
+
+	return &buffer, count, nil
+}
+
 func loadfile(fp string) (*bytes.Buffer, error) {
 	f, err := openFile(fp)
 	if err != nil {

slice/validate.go

@@ -1,17 +1,59 @@
 package slice
 
 import (
+	"bytes"
 	"fmt"
+	"path/filepath"
 	"regexp"
 )
 
-var regKN = regexp.MustCompile(`^[^/]+/[^/]+$`)
+var (
+	regKN      = regexp.MustCompile(`^[^/]+/[^/]+$`)
+	extensions = []string{".yaml", ".yml"}
+)
 
 func (s *Split) init() error {
-	s.log.Printf("Loading file %s", s.opts.InputFile)
-	buf, err := loadfile(s.opts.InputFile)
-	if err != nil {
-		return err
+	s.log.Printf("Initializing with settings: %#v", s.opts)
+
+	if s.opts.InputFile != "" && s.opts.InputFolder != "" {
+		return fmt.Errorf("cannot specify both input file and input folder")
+	}
+
+	if s.opts.InputFile == "" && s.opts.InputFolder == "" {
+		return fmt.Errorf("input file or input folder is required")
+	}
+
+	var buf *bytes.Buffer
+
+	if s.opts.InputFile != "" {
+		s.log.Printf("Loading file %s", s.opts.InputFile)
+		var err error
+		buf, err = loadfile(s.opts.InputFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if s.opts.InputFolder != "" {
+		exts := extensions
+		s.opts.InputFolder = filepath.Clean(s.opts.InputFolder)
+
+		if len(s.opts.InputFolderExt) > 0 {
+			exts = s.opts.InputFolderExt
+		}
+
+		s.log.Printf("Loading folder %q", s.opts.InputFolder)
+		var err error
+		var count int
+		buf, count, err = loadfolder(exts, s.opts.InputFolder, s.opts.Recurse)
+		if err != nil {
+			return err
+		}
+		s.log.Printf("Found %d files in folder %q", count, s.opts.InputFolder)
+	}
+
+	if buf == nil || buf.Len() == 0 {
+		return fmt.Errorf("no data found in input file or folder")
 	}
 
 	s.data = buf