Merge pull request #348 from b4nst/feat/local-module-in-bundle

Add support for file based module in bundle

Makefile

@@ -48,6 +48,7 @@ cue-vet: build ## Vet and fmt CUE files.
 	done
 	./bin/timoni mod vet ./cmd/timoni/testdata/module
 	./bin/timoni mod vet ./internal/engine/testdata/module
+	./bin/timoni mod vet ./internal/engine/fetcher/testdata/module
 	cue fmt ./internal/engine/testdata/module-values
 
 REDIS_VER=$(shell grep 'tag:' examples/redis/values.cue | awk '{ print $$2 }' | tr -d '"')

api/v1alpha1/artifact.go

@@ -20,6 +20,9 @@ const (
 	// ArtifactPrefix is the prefix used for OpenContainers artifact address.
 	ArtifactPrefix = "oci://"
 
+	// LocalPrefix is the prefix used for local file system address.
+	LocalPrefix = "file://"
+
 	// UserAgent is the agent name used for OpenContainers artifact operations.
 	UserAgent = "timoni/v1"
 

api/v1alpha1/bundle.go

@@ -55,7 +55,7 @@ import "strings"
 	name:       string & =~"^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" & strings.MaxRunes(63) & strings.MinRunes(1)
 	instances: [string & =~"^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" & strings.MaxRunes(63) & strings.MinRunes(1)]: {
 		module: close({
-			url:     string & =~"^oci://.*$"
+			url:     string & =~"^(oci|file)://.*$"
 			version: *"latest" | string
 			digest?: string
 		})

api/v1alpha1/module.go

@@ -26,8 +26,8 @@ type ModuleReference struct {
 	// Name of the module.
 	Name string `json:"name"`
 
-	// Repository is the OCI artifact repo name in the format
-	// 'oci://<reg.host>/<org>/<repo>'.
+	// Repository is the OCI artifact or local repo name in the format
+	// 'oci://<reg.host>/<org>/<repo>' or 'file://<path>'.
 	Repository string `json:"repository"`
 
 	// Version is the OCI artifact tag in strict semver format.

cmd/timoni/apply.go

@@ -32,6 +32,7 @@ import (
 
 	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
+	"github.com/stefanprodan/timoni/internal/engine/fetcher"
 	"github.com/stefanprodan/timoni/internal/flags"
 	"github.com/stefanprodan/timoni/internal/runtime"
 )
@@ -163,16 +164,19 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	fetcher := engine.NewFetcher(
-		ctxPull,
-		applyArgs.module,
-		version,
-		tmpDir,
-		rootArgs.cacheDir,
-		applyArgs.creds.String(),
-		rootArgs.registryInsecure,
-	)
-	mod, err := fetcher.Fetch()
+	f, err := fetcher.New(ctxPull, fetcher.Options{
+		Source:       applyArgs.module,
+		Version:      version,
+		Destination:  tmpDir,
+		CacheDir:     rootArgs.cacheDir,
+		Creds:        applyArgs.creds.String(),
+		Insecure:     rootArgs.registryInsecure,
+		DefaultLocal: true,
+	})
+	if err != nil {
+		return err
+	}
+	mod, err := f.Fetch()
 	if err != nil {
 		return err
 	}
@@ -182,7 +186,7 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 		cuectx,
 		applyArgs.name,
 		*kubeconfigArgs.Namespace,
-		fetcher.GetModuleRoot(),
+		f.GetModuleRoot(),
 		applyArgs.pkg.String(),
 	)
 
@@ -217,7 +221,7 @@ func runApplyCmd(cmd *cobra.Command, args []string) error {
 
 	buildResult, err := builder.Build()
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "build failed", err)
+		return describeErr(f.GetModuleRoot(), "build failed", err)
 	}
 
 	finalValues, err := builder.GetDefaultValues()

cmd/timoni/build.go

@@ -38,6 +38,7 @@ import (
 
 	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
+	"github.com/stefanprodan/timoni/internal/engine/fetcher"
 	"github.com/stefanprodan/timoni/internal/flags"
 )
 
@@ -114,16 +115,19 @@ func runBuildCmd(cmd *cobra.Command, args []string) error {
 	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	fetcher := engine.NewFetcher(
-		ctxPull,
-		buildArgs.module,
-		version,
-		tmpDir,
-		rootArgs.cacheDir,
-		buildArgs.creds.String(),
-		rootArgs.registryInsecure,
-	)
-	mod, err := fetcher.Fetch()
+	f, err := fetcher.New(ctxPull, fetcher.Options{
+		Source:       buildArgs.module,
+		Version:      version,
+		Destination:  tmpDir,
+		CacheDir:     rootArgs.cacheDir,
+		Creds:        buildArgs.creds.String(),
+		Insecure:     rootArgs.registryInsecure,
+		DefaultLocal: true,
+	})
+	if err != nil {
+		return err
+	}
+	mod, err := f.Fetch()
 	if err != nil {
 		return err
 	}
@@ -132,7 +136,7 @@ func runBuildCmd(cmd *cobra.Command, args []string) error {
 		ctx,
 		buildArgs.name,
 		*kubeconfigArgs.Namespace,
-		fetcher.GetModuleRoot(),
+		f.GetModuleRoot(),
 		buildArgs.pkg.String(),
 	)
 
@@ -158,7 +162,7 @@ func runBuildCmd(cmd *cobra.Command, args []string) error {
 
 	buildResult, err := builder.Build()
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "build failed", err)
+		return describeErr(f.GetModuleRoot(), "build failed", err)
 	}
 
 	apiVer, err := builder.GetAPIVersion(buildResult)

cmd/timoni/bundle_apply.go

@@ -36,6 +36,7 @@ import (
 
 	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
+	"github.com/stefanprodan/timoni/internal/engine/fetcher"
 	"github.com/stefanprodan/timoni/internal/flags"
 	"github.com/stefanprodan/timoni/internal/runtime"
 )
@@ -253,16 +254,19 @@ func fetchBundleInstanceModule(ctx context.Context, instance *engine.BundleInsta
 		moduleVersion = "@" + instance.Module.Digest
 	}
 
-	fetcher := engine.NewFetcher(
-		ctx,
-		instance.Module.Repository,
-		moduleVersion,
-		modDir,
-		rootArgs.cacheDir,
-		bundleApplyArgs.creds.String(),
-		rootArgs.registryInsecure,
-	)
-	mod, err := fetcher.Fetch()
+	f, err := fetcher.New(ctx, fetcher.Options{
+		Source:      instance.Module.Repository,
+		Version:     moduleVersion,
+		Destination: modDir,
+		CacheDir:    rootArgs.cacheDir,
+		Creds:       bundleApplyArgs.creds.String(),
+		Insecure:    rootArgs.registryInsecure,
+	})
+	if err != nil {
+		return err
+	}
+
+	mod, err := f.Fetch()
 	if err != nil {
 		return err
 	}

cmd/timoni/mod_show_config.go

@@ -34,6 +34,7 @@ import (
 	"github.com/spf13/cobra"
 	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
+	"github.com/stefanprodan/timoni/internal/engine/fetcher"
 	"github.com/stefanprodan/timoni/internal/flags"
 )
 
@@ -89,16 +90,19 @@ func runConfigShowModCmd(cmd *cobra.Command, args []string) error {
 	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	fetcher := engine.NewFetcher(
-		ctxPull,
-		configShowModArgs.path,
-		apiv1.LatestVersion,
-		tmpDir,
-		rootArgs.cacheDir,
-		"",
-		rootArgs.registryInsecure,
-	)
-	mod, err := fetcher.Fetch()
+	f, err := fetcher.New(ctxPull, fetcher.Options{
+		Source:       configShowModArgs.path,
+		Version:      apiv1.LatestVersion,
+		Destination:  tmpDir,
+		CacheDir:     rootArgs.cacheDir,
+		Insecure:     rootArgs.registryInsecure,
+		DefaultLocal: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	mod, err := f.Fetch()
 	if err != nil {
 		return err
 	}
@@ -107,7 +111,7 @@ func runConfigShowModCmd(cmd *cobra.Command, args []string) error {
 		cuectx,
 		configShowModArgs.name,
 		*kubeconfigArgs.Namespace,
-		fetcher.GetModuleRoot(),
+		f.GetModuleRoot(),
 		configShowModArgs.pkg.String(),
 	)
 
@@ -122,34 +126,34 @@ func runConfigShowModCmd(cmd *cobra.Command, args []string) error {
 
 	buildResult, err := builder.Build()
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "validation failed", err)
+		return describeErr(f.GetModuleRoot(), "validation failed", err)
 	}
 
 	rows, err := builder.GetConfigDoc(buildResult)
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "failed to get config structure", err)
+		return describeErr(f.GetModuleRoot(), "failed to get config structure", err)
 	}
 
 	header := []string{"Key", "Type", "Default", "Description"}
 
 	if configShowModArgs.output == "" {
 		printMarkDownTable(rootCmd.OutOrStdout(), header, rows)
 	} else {
-		tmpFile, err := writeFile(configShowModArgs.output, header, rows, fetcher)
+		tmpFile, err := writeFile(configShowModArgs.output, header, rows, f)
 		if err != nil {
 			return err
 		}
 
 		err = os.Rename(tmpFile, configShowModArgs.output)
 		if err != nil {
-			return describeErr(fetcher.GetModuleRoot(), "Unable to rename file", err)
+			return describeErr(f.GetModuleRoot(), "Unable to rename file", err)
 		}
 	}
 
 	return nil
 }
 
-func writeFile(readFile string, header []string, rows [][]string, fetcher *engine.Fetcher) (string, error) {
+func writeFile(readFile string, header []string, rows [][]string, f fetcher.Fetcher) (string, error) {
 	// Generate the markdown table
 	var tableBuffer bytes.Buffer
 	tableWriter := bufio.NewWriter(&tableBuffer)
@@ -164,18 +168,18 @@ func writeFile(readFile string, header []string, rows [][]string, fetcher *engin
 			inputFile, err = os.Create(readFile)
 
 			if err != nil {
-				return "", describeErr(fetcher.GetModuleRoot(), "Unable to create the temporary output file", err)
+				return "", describeErr(f.GetModuleRoot(), "Unable to create the temporary output file", err)
 			}
 		} else {
-			return "", describeErr(fetcher.GetModuleRoot(), "Unable to create the temporary output file", err)
+			return "", describeErr(f.GetModuleRoot(), "Unable to create the temporary output file", err)
 		}
 	}
 	defer inputFile.Close()
 
 	// open the output file
 	outputFile, err := os.Create(tmpFileName)
 	if err != nil {
-		return "", describeErr(fetcher.GetModuleRoot(), "Unable to create the temporary output file", err)
+		return "", describeErr(f.GetModuleRoot(), "Unable to create the temporary output file", err)
 	}
 	defer outputFile.Close()
 
@@ -196,7 +200,7 @@ func writeFile(readFile string, header []string, rows [][]string, fetcher *engin
 
 			matched, err := regexp.MatchString(`^\|.*\|$`, line)
 			if err != nil {
-				return "", describeErr(fetcher.GetModuleRoot(), "Regex Match for table content failed", err)
+				return "", describeErr(f.GetModuleRoot(), "Regex Match for table content failed", err)
 			}
 
 			if configSection && !foundTable && matched {
@@ -220,7 +224,7 @@ func writeFile(readFile string, header []string, rows [][]string, fetcher *engin
 
 	err = outputWriter.Flush()
 	if err != nil {
-		return "", describeErr(fetcher.GetModuleRoot(), "Failed to Flush Writer", err)
+		return "", describeErr(f.GetModuleRoot(), "Failed to Flush Writer", err)
 	}
 
 	return tmpFileName, nil

cmd/timoni/mod_vet.go

@@ -32,6 +32,7 @@ import (
 
 	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
+	"github.com/stefanprodan/timoni/internal/engine/fetcher"
 	"github.com/stefanprodan/timoni/internal/flags"
 )
 
@@ -92,16 +93,19 @@ func runVetModCmd(cmd *cobra.Command, args []string) error {
 	ctxPull, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	fetcher := engine.NewFetcher(
-		ctxPull,
-		vetModArgs.path,
-		apiv1.LatestVersion,
-		tmpDir,
-		rootArgs.cacheDir,
-		"",
-		rootArgs.registryInsecure,
-	)
-	mod, err := fetcher.Fetch()
+	f, err := fetcher.New(ctxPull, fetcher.Options{
+		Source:       vetModArgs.path,
+		Version:      apiv1.LatestVersion,
+		Destination:  tmpDir,
+		CacheDir:     rootArgs.cacheDir,
+		Insecure:     rootArgs.registryInsecure,
+		DefaultLocal: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	mod, err := f.Fetch()
 	if err != nil {
 		return err
 	}
@@ -126,7 +130,7 @@ func runVetModCmd(cmd *cobra.Command, args []string) error {
 		cuectx,
 		vetModArgs.name,
 		*kubeconfigArgs.Namespace,
-		fetcher.GetModuleRoot(),
+		f.GetModuleRoot(),
 		vetModArgs.pkg.String(),
 	)
 
@@ -152,7 +156,7 @@ func runVetModCmd(cmd *cobra.Command, args []string) error {
 
 	buildResult, err := builder.Build(tags...)
 	if err != nil {
-		return describeErr(fetcher.GetModuleRoot(), "validation failed", err)
+		return describeErr(f.GetModuleRoot(), "validation failed", err)
 	}
 
 	applySets, err := builder.GetApplySets(buildResult)

docs/bundle.md

@@ -242,8 +242,16 @@ of the instance's module.
 
 #### URL
 
-The `instance.module.url` is a required field that specifies the OCI repository address
-where the module is published. The `url` field must be in the format `oci://<registry-host>/<repo-name>`.
+The `instance.module.url` is a required field that specifies the source of the module.
+It can be either an OCI repository address (preferred) or a local path to a module (useful during development).
+
+When using an OCI repository, the `url` field must be in the format `oci://<registry-host>/<repo-name>`.
+
+When using a Local path, the `url` field must be in the format `file://path/to/module`.
+
+!!! tip Relative paths
+
+    Relative paths are always computed relatively to the path of the bundle file containing the value.
 
 #### Version