added remote sync updates

.github/workflows/tests.yaml

@@ -258,7 +258,7 @@ jobs:
           # download local helm repository
           curl -sfOL https://github.com/rancherfederal/rancher-cluster-templates/releases/download/rancher-cluster-templates-0.5.2/rancher-cluster-templates-0.5.2.tgz
           # verify via sync
-          hauler store sync --files testdata/hauler-manifest-pipeline.yaml
+          hauler store sync --filename testdata/hauler-manifest-pipeline.yaml
           # need more tests here
 
       - name: Verify - hauler store serve

cmd/hauler/cli/store.go

@@ -67,6 +67,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 	cmd := &cobra.Command{
 		Use:   "sync",
 		Short: "Sync content to the content store",
+		Args:  cobra.ExactArgs(0),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
@@ -75,7 +76,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 				return err
 			}
 
-			return store.SyncCmd(ctx, o, s, rso, ro)
+			return store.SyncCmd(ctx, o, s, rso, ro, o.TempOverride)
 		},
 	}
 	o.AddFlags(cmd)

cmd/hauler/cli/store/add.go

@@ -42,13 +42,13 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error {
 		return err
 	}
 
-	l.Infof("adding 'file' [%s] to the store as [%s]", fi.Path, ref.Name())
+	l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name())
 	_, err = s.AddOCI(ctx, f, ref.Name())
 	if err != nil {
 		return err
 	}
 
-	l.Infof("successfully added 'file' [%s]", ref.Name())
+	l.Infof("successfully added file [%s]", ref.Name())
 
 	return nil
 }
@@ -83,31 +83,31 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform
 		}
 	}
 
-	l.Infof("adding 'image' [%s] to the store", i.Name)
+	l.Infof("adding image [%s] to the store", i.Name)
 
 	r, err := name.ParseReference(i.Name)
 	if err != nil {
 		if ro.IgnoreErrors {
-			l.Warnf("unable to parse 'image' [%s]: %v... skipping...", i.Name, err)
+			l.Warnf("unable to parse image [%s]: %v... skipping...", i.Name, err)
 			return nil
 		} else {
-			l.Errorf("unable to parse 'image' [%s]: %v", i.Name, err)
+			l.Errorf("unable to parse image [%s]: %v", i.Name, err)
 			return err
 		}
 	}
 
 	err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
 	if err != nil {
 		if ro.IgnoreErrors {
-			l.Warnf("unable to add 'image' [%s] to store: %v... skipping...", r.Name(), err)
+			l.Warnf("unable to add image [%s] to store: %v... skipping...", r.Name(), err)
 			return nil
 		} else {
-			l.Errorf("unable to add 'image' [%s] to store: %v", r.Name(), err)
+			l.Errorf("unable to add image [%s] to store: %v", r.Name(), err)
 			return err
 		}
 	}
 
-	l.Infof("successfully added 'image' [%s]", r.Name())
+	l.Infof("successfully added image [%s]", r.Name())
 	return nil
 }
 
@@ -125,7 +125,7 @@ func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, ch
 func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts *action.ChartPathOptions) error {
 	l := log.FromContext(ctx)
 
-	l.Infof("adding 'chart' [%s] to the store", cfg.Name)
+	l.Infof("adding chart [%s] to the store", cfg.Name)
 
 	// TODO: This shouldn't be necessary
 	opts.RepoURL = cfg.RepoURL
@@ -150,6 +150,6 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts *
 		return err
 	}
 
-	l.Infof("successfully added 'chart' [%s]", ref.Name())
+	l.Infof("successfully added chart [%s]", ref.Name())
 	return nil
 }

cmd/hauler/cli/store/sync.go

@@ -5,7 +5,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mitchellh/go-homedir"
@@ -14,6 +16,7 @@ import (
 
 	"hauler.dev/go/hauler/internal/flags"
 	"hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1"
+	"hauler.dev/go/hauler/pkg/artifacts/file/getter"
 	tchart "hauler.dev/go/hauler/pkg/collection/chart"
 	"hauler.dev/go/hauler/pkg/collection/imagetxt"
 	"hauler.dev/go/hauler/pkg/consts"
@@ -24,9 +27,21 @@ import (
 	"hauler.dev/go/hauler/pkg/store"
 )
 
-func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
+func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, tempOverride string) error {
 	l := log.FromContext(ctx)
 
+	if tempOverride == "" {
+		tempOverride = os.Getenv(consts.HaulerTempDir)
+	}
+
+	tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tempDir)
+
+	l.Debugf("using temporary directory at [%s]", tempDir)
+
 	// if passed products, check for a remote manifest to retrieve and use.
 	for _, product := range o.Products {
 		l.Infof("processing content file for product [%s]", product)
@@ -64,10 +79,22 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
 		}
 	}
 
-	// if passed a local manifest, process it
-	for _, filename := range o.ContentFiles {
-		l.Debugf("processing content file: [%s]", filename)
-		fi, err := os.Open(filename)
+	// if passed a manifest... process it
+	for _, filename := range o.Filename {
+		l.Infof("processing manifest [%s] to store [%s]", filename, o.StoreDir)
+		var localFilename string
+
+		if strings.HasPrefix(filename, "http://") || strings.HasPrefix(filename, "https://") {
+			l.Debugf("detected remote manifest... starting download... [%s]", filename)
+			var err error
+			localFilename, err = downloadRemote(ctx, filename, tempDir)
+			if err != nil {
+				return err
+			}
+		} else {
+			localFilename = filename
+		}
+		fi, err := os.Open(localFilename)
 		if err != nil {
 			return err
 		}
@@ -105,7 +132,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 			continue
 		}
 
-		l.Infof("syncing [%s] to store", obj.GroupVersionKind().String())
+		l.Infof("syncing [%s]", obj.GroupVersionKind().String())
 
 		// TODO: Should type switch instead...
 		switch obj.GroupVersionKind().Kind {
@@ -256,3 +283,35 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
 	}
 	return nil
 }
+
+// downloadRemote downloads the remote file using the existing getter
+func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) {
+	parsedURL, err := url.Parse(remoteURL)
+	if err != nil {
+		return "", err
+	}
+	h := getter.NewHttp()
+	rc, err := h.Open(ctx, parsedURL)
+	if err != nil {
+		return "", err
+	}
+	defer rc.Close()
+
+	fileName := h.Name(parsedURL)
+	if fileName == "" {
+		fileName = filepath.Base(parsedURL.Path)
+	}
+
+	localPath := filepath.Join(tempDirDest, fileName)
+	out, err := os.Create(localPath)
+	if err != nil {
+		return "", err
+	}
+	defer out.Close()
+
+	if _, err = io.Copy(out, rc); err != nil {
+		return "", err
+	}
+
+	return localPath, nil
+}

internal/flags/store.go

@@ -41,6 +41,8 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) {
 		return nil, err
 	}
 
+	o.StoreDir = abs
+
 	l.Debugf("using store at [%s]", abs)
 
 	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {

internal/flags/sync.go

@@ -1,24 +1,29 @@
 package flags
 
-import "github.com/spf13/cobra"
+import (
+	"github.com/spf13/cobra"
+	"hauler.dev/go/hauler/pkg/consts"
+)
 
 type SyncOpts struct {
 	*StoreRootOpts
-	ContentFiles    []string
+	Filename        []string
 	Key             string
 	Products        []string
 	Platform        string
 	Registry        string
 	ProductRegistry string
+	TempOverride    string
 }
 
 func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
 	f := cmd.Flags()
 
-	f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Location of content manifests (files)... i.e. --files ./rke2-files.yaml")
+	f.StringSliceVarP(&o.Filename, "filename", "f", []string{consts.DefaultHaulerManifestName}, "(Optional) Specify the name of inputted manifest(s)")
 	f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification")
 	f.StringSliceVar(&o.Products, "products", []string{}, "(Optional) Specify the product name to fetch collections from the product registry i.e. rancher=v2.8.5,rke2=v1.28.11+rke2r1")
 	f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)")
 	f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one")
 	f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
+	f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS")
 }

pkg/consts/consts.go

@@ -72,20 +72,21 @@ const (
 	ImageConfigFile   = "config.json"
 
 	// other constraints
-	CarbideRegistry          = "rgcrprod.azurecr.us"
-	APIVersion               = "v1alpha1"
-	DefaultNamespace         = "hauler"
-	DefaultTag               = "latest"
-	DefaultStoreName         = "store"
-	DefaultHaulerDirName     = ".hauler"
-	DefaultHaulerTempDirName = "hauler"
-	DefaultRegistryRootDir   = "registry"
-	DefaultRegistryPort      = 5000
-	DefaultFileserverRootDir = "fileserver"
-	DefaultFileserverPort    = 8080
-	DefaultFileserverTimeout = 60
-	DefaultHaulArchiveName   = "haul.tar.zst"
-	DefaultRetries           = 3
-	RetriesInterval          = 5
-	CustomTimeFormat         = "2006-01-02 15:04:05"
+	CarbideRegistry           = "rgcrprod.azurecr.us"
+	APIVersion                = "v1alpha1"
+	DefaultNamespace          = "hauler"
+	DefaultTag                = "latest"
+	DefaultStoreName          = "store"
+	DefaultHaulerDirName      = ".hauler"
+	DefaultHaulerTempDirName  = "hauler"
+	DefaultRegistryRootDir    = "registry"
+	DefaultRegistryPort       = 5000
+	DefaultFileserverRootDir  = "fileserver"
+	DefaultFileserverPort     = 8080
+	DefaultFileserverTimeout  = 60
+	DefaultHaulArchiveName    = "haul.tar.zst"
+	DefaultHaulerManifestName = "hauler-manifest.yaml"
+	DefaultRetries            = 3
+	RetriesInterval           = 5
+	CustomTimeFormat          = "2006-01-02 15:04:05"
 )