diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 10a8ec1..d989ba6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -258,7 +258,7 @@ jobs: # download local helm repository curl -sfOL https://github.com/rancherfederal/rancher-cluster-templates/releases/download/rancher-cluster-templates-0.5.2/rancher-cluster-templates-0.5.2.tgz # verify via sync - hauler store sync --files testdata/hauler-manifest-pipeline.yaml + hauler store sync --filename testdata/hauler-manifest-pipeline.yaml # need more tests here - name: Verify - hauler store serve diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 754439e..edc650b 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -67,6 +67,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman cmd := &cobra.Command{ Use: "sync", Short: "Sync content to the content store", + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() @@ -75,7 +76,7 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman return err } - return store.SyncCmd(ctx, o, s, rso, ro) + return store.SyncCmd(ctx, o, s, rso, ro, o.TempOverride) }, } o.AddFlags(cmd) @@ -123,7 +124,6 @@ func addStoreServe(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comma return cmd } -// RegistryCmd serves the registry func addStoreServeRegistry(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command { o := &flags.ServeRegistryOpts{StoreRootOpts: rso} @@ -147,7 +147,6 @@ func addStoreServeRegistry(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cob return cmd } -// FileServerCmd serves the file server func addStoreServeFiles(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command { o := &flags.ServeFilesOpts{StoreRootOpts: rso} diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index b4a22d6..505362a 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -42,13 +42,13 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1alpha1.File) error { return err } - l.Infof("adding 'file' [%s] to the store as [%s]", fi.Path, ref.Name()) + l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name()) _, err = s.AddOCI(ctx, f, ref.Name()) if err != nil { return err } - l.Infof("successfully added 'file' [%s]", ref.Name()) + l.Infof("successfully added file [%s]", ref.Name()) return nil } @@ -83,15 +83,15 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform } } - l.Infof("adding 'image' [%s] to the store", i.Name) + l.Infof("adding image [%s] to the store", i.Name) r, err := name.ParseReference(i.Name) if err != nil { if ro.IgnoreErrors { - l.Warnf("unable to parse 'image' [%s]: %v... skipping...", i.Name, err) + l.Warnf("unable to parse image [%s]: %v... skipping...", i.Name, err) return nil } else { - l.Errorf("unable to parse 'image' [%s]: %v", i.Name, err) + l.Errorf("unable to parse image [%s]: %v", i.Name, err) return err } } @@ -99,15 +99,15 @@ func storeImage(ctx context.Context, s *store.Layout, i v1alpha1.Image, platform err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro) if err != nil { if ro.IgnoreErrors { - l.Warnf("unable to add 'image' [%s] to store: %v... skipping...", r.Name(), err) + l.Warnf("unable to add image [%s] to store: %v... skipping...", r.Name(), err) return nil } else { - l.Errorf("unable to add 'image' [%s] to store: %v", r.Name(), err) + l.Errorf("unable to add image [%s] to store: %v", r.Name(), err) return err } } - l.Infof("successfully added 'image' [%s]", r.Name()) + l.Infof("successfully added image [%s]", r.Name()) return nil } @@ -125,7 +125,7 @@ func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, ch func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts *action.ChartPathOptions) error { l := log.FromContext(ctx) - l.Infof("adding 'chart' [%s] to the store", cfg.Name) + l.Infof("adding chart [%s] to the store", cfg.Name) // TODO: This shouldn't be necessary opts.RepoURL = cfg.RepoURL @@ -150,6 +150,6 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1alpha1.Chart, opts * return err } - l.Infof("successfully added 'chart' [%s]", ref.Name()) + l.Infof("successfully added chart [%s]", ref.Name()) return nil } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index a4e913c..67f30c5 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -5,7 +5,9 @@ import ( "context" "fmt" "io" + "net/url" "os" + "path/filepath" "strings" "github.com/mitchellh/go-homedir" @@ -14,6 +16,7 @@ import ( "hauler.dev/go/hauler/internal/flags" "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "hauler.dev/go/hauler/pkg/artifacts/file/getter" tchart "hauler.dev/go/hauler/pkg/collection/chart" "hauler.dev/go/hauler/pkg/collection/imagetxt" "hauler.dev/go/hauler/pkg/consts" @@ -24,9 +27,21 @@ import ( "hauler.dev/go/hauler/pkg/store" ) -func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error { +func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, tempOverride string) error { l := log.FromContext(ctx) + if tempOverride == "" { + tempOverride = os.Getenv(consts.HaulerTempDir) + } + + tempDir, err := os.MkdirTemp(tempOverride, consts.DefaultHaulerTempDirName) + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + l.Debugf("using temporary directory at [%s]", tempDir) + // if passed products, check for a remote manifest to retrieve and use. for _, product := range o.Products { l.Infof("processing content file for product [%s]", product) @@ -62,12 +77,25 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags if err != nil { return err } + l.Infof("processing completed successfully") } - // if passed a local manifest, process it - for _, filename := range o.ContentFiles { - l.Debugf("processing content file: [%s]", filename) - fi, err := os.Open(filename) + // if passed a manifest... process it + for _, filename := range o.Filename { + l.Infof("processing manifest [%s] to store [%s]", filename, o.StoreDir) + var localFilename string + + if strings.HasPrefix(filename, "http://") || strings.HasPrefix(filename, "https://") { + l.Debugf("detected remote manifest... starting download... [%s]", filename) + var err error + localFilename, err = downloadRemote(ctx, filename, tempDir) + if err != nil { + return err + } + } else { + localFilename = filename + } + fi, err := os.Open(localFilename) if err != nil { return err } @@ -75,6 +103,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags if err != nil { return err } + l.Infof("processing completed successfully") } return nil @@ -105,7 +134,7 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor continue } - l.Infof("syncing [%s] to store", obj.GroupVersionKind().String()) + l.Infof("syncing [%s]", obj.GroupVersionKind().String()) // TODO: Should type switch instead... switch obj.GroupVersionKind().Kind { @@ -256,3 +285,35 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor } return nil } + +// downloadRemote downloads the remote file using the existing getter +func downloadRemote(ctx context.Context, remoteURL, tempDirDest string) (string, error) { + parsedURL, err := url.Parse(remoteURL) + if err != nil { + return "", err + } + h := getter.NewHttp() + rc, err := h.Open(ctx, parsedURL) + if err != nil { + return "", err + } + defer rc.Close() + + fileName := h.Name(parsedURL) + if fileName == "" { + fileName = filepath.Base(parsedURL.Path) + } + + localPath := filepath.Join(tempDirDest, fileName) + out, err := os.Create(localPath) + if err != nil { + return "", err + } + defer out.Close() + + if _, err = io.Copy(out, rc); err != nil { + return "", err + } + + return localPath, nil +} diff --git a/internal/flags/store.go b/internal/flags/store.go index c2dd6ce..892384c 100644 --- a/internal/flags/store.go +++ b/internal/flags/store.go @@ -41,6 +41,8 @@ func (o *StoreRootOpts) Store(ctx context.Context) (*store.Layout, error) { return nil, err } + o.StoreDir = abs + l.Debugf("using store at [%s]", abs) if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) { diff --git a/internal/flags/sync.go b/internal/flags/sync.go index 00a5ed6..355ef4b 100644 --- a/internal/flags/sync.go +++ b/internal/flags/sync.go @@ -1,24 +1,29 @@ package flags -import "github.com/spf13/cobra" +import ( + "github.com/spf13/cobra" + "hauler.dev/go/hauler/pkg/consts" +) type SyncOpts struct { *StoreRootOpts - ContentFiles []string + Filename []string Key string Products []string Platform string Registry string ProductRegistry string + TempOverride string } func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f := cmd.Flags() - f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Location of content manifests (files)... i.e. --files ./rke2-files.yaml") + f.StringSliceVarP(&o.Filename, "filename", "f", []string{consts.DefaultHaulerManifestName}, "Specify the name of manifest(s) to sync") f.StringVarP(&o.Key, "key", "k", "", "(Optional) Location of public key to use for signature verification") f.StringSliceVar(&o.Products, "products", []string{}, "(Optional) Specify the product name to fetch collections from the product registry i.e. rancher=v2.8.5,rke2=v1.28.11+rke2r1") f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specify the platform of the image... i.e linux/amd64 (defaults to all)") f.StringVarP(&o.Registry, "registry", "g", "", "(Optional) Specify the registry of the image for images that do not alredy define one") f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)") + f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS") } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index df2231b..66775e6 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -72,20 +72,21 @@ const ( ImageConfigFile = "config.json" // other constraints - CarbideRegistry = "rgcrprod.azurecr.us" - APIVersion = "v1alpha1" - DefaultNamespace = "hauler" - DefaultTag = "latest" - DefaultStoreName = "store" - DefaultHaulerDirName = ".hauler" - DefaultHaulerTempDirName = "hauler" - DefaultRegistryRootDir = "registry" - DefaultRegistryPort = 5000 - DefaultFileserverRootDir = "fileserver" - DefaultFileserverPort = 8080 - DefaultFileserverTimeout = 60 - DefaultHaulArchiveName = "haul.tar.zst" - DefaultRetries = 3 - RetriesInterval = 5 - CustomTimeFormat = "2006-01-02 15:04:05" + CarbideRegistry = "rgcrprod.azurecr.us" + APIVersion = "v1alpha1" + DefaultNamespace = "hauler" + DefaultTag = "latest" + DefaultStoreName = "store" + DefaultHaulerDirName = ".hauler" + DefaultHaulerTempDirName = "hauler" + DefaultRegistryRootDir = "registry" + DefaultRegistryPort = 5000 + DefaultFileserverRootDir = "fileserver" + DefaultFileserverPort = 8080 + DefaultFileserverTimeout = 60 + DefaultHaulArchiveName = "haul.tar.zst" + DefaultHaulerManifestName = "hauler-manifest.yaml" + DefaultRetries = 3 + RetriesInterval = 5 + CustomTimeFormat = "2006-01-02 15:04:05" )