Add option to publish via a different URI

Signed-off-by: Steven Sheehy <[email protected]>

README.md

@@ -117,6 +117,20 @@ To work with this repo by it's name, first you need to add it using native helm
 
     $ helm repo add mynewrepo s3://bucket-name/charts
 
+To store the repository in S3, but to publish it at a different URI you can specify the
+URI when initializing. This will rewrite the URIs in the repository index so that they are
+downloadable via the published URI. This is useful in case you want to keep the S3
+bucket private and expose the repository over HTTP(S) via CloudFront or a web server.
+The delete and push commands will automatically take into account this publish URI when
+updating the index.
+
+    $ helm s3 init s3://bucket-name/charts --publish https://charts.my-company.tld
+
+The repository owner can continue to use the S3 URIs while the repository user can
+consume the repository via its published URI:
+
+   $ helm repo add publishedrepo https://charts.my-company.tld
+
 ### Push
 
 Now you can push your chart to this repo:
@@ -166,6 +180,10 @@ the index in accordance with the charts in the repository.
 
     $ helm s3 reindex mynewrepo
 
+Alternatively, you can specify `--publish uri` to reindex an existing repository with a
+different published URI. If you don't specify a publish URI, it will reset the repository
+back to using S3 URIs.
+
 ## Uninstall
 
     $ helm plugin remove s3

cmd/helms3/delete.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/helm/pkg/repo"
 
 	"github.com/hypnoglow/helm-s3/internal/awss3"
 	"github.com/hypnoglow/helm-s3/internal/awsutil"
@@ -29,7 +31,8 @@ func (act deleteAction) Run(ctx context.Context) error {
 	storage := awss3.New(sess)
 
 	// Fetch current index.
-	b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml")
+	indexURI := repoEntry.URL + "/index.yaml"
+	b, err := storage.FetchRaw(ctx, indexURI)
 	if err != nil {
 		return errors.WithMessage(err, "fetch current repo index")
 	}
@@ -56,16 +59,35 @@ func (act deleteAction) Run(ctx context.Context) error {
 	if len(chartVersion.URLs) < 1 {
 		return fmt.Errorf("chart version index record has no urls")
 	}
-	uri := chartVersion.URLs[0]
+
+	metadata, err := storage.GetMetadata(ctx, indexURI)
+	if err != nil {
+		return err
+	}
+
+	publishURI := metadata[strings.Title(awss3.MetaPublishURI)]
+	uri := fmt.Sprintf("%s/%s-%s.tgz", repoEntry.URL, chartVersion.Metadata.Name, chartVersion.Metadata.Version)
 
 	if err := storage.Delete(ctx, uri); err != nil {
 		return errors.WithMessage(err, "delete chart file from s3")
 	}
-	if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, idxReader); err != nil {
+	if err := storage.PutIndex(ctx, repoEntry.URL, publishURI, act.acl, idxReader); err != nil {
 		return errors.WithMessage(err, "upload new index to s3")
 	}
 
-	if err := idx.WriteFile(repoEntry.Cache, 0644); err != nil {
+	localIndexFile, err := repo.LoadIndexFile(repoEntry.Cache)
+	if err != nil {
+		return err
+	}
+
+	localIndex := &index.Index{IndexFile: localIndexFile}
+
+	_, err = localIndex.Delete(act.name, act.version)
+	if err != nil {
+		return errors.WithMessage(err, "delete chart from local index")
+	}
+
+	if err := localIndex.WriteFile(repoEntry.Cache, 0644); err != nil {
 		return errors.WithMessage(err, "update local index")
 	}
 

cmd/helms3/init.go

@@ -11,8 +11,9 @@ import (
 )
 
 type initAction struct {
-	uri string
-	acl string
+	uri        string
+	publishURI string
+	acl        string
 }
 
 func (act initAction) Run(ctx context.Context) error {
@@ -27,7 +28,7 @@ func (act initAction) Run(ctx context.Context) error {
 	}
 	storage := awss3.New(sess)
 
-	if err := storage.PutIndex(ctx, act.uri, act.acl, r); err != nil {
+	if err := storage.PutIndex(ctx, act.uri, act.publishURI, act.acl, r); err != nil {
 		return errors.WithMessage(err, "upload index to s3")
 	}
 

cmd/helms3/main.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"time"
 
-	"gopkg.in/alecthomas/kingpin.v2"
+	kingpin "gopkg.in/alecthomas/kingpin.v2"
 )
 
 var (
@@ -76,6 +76,9 @@ func main() {
 	initURI := initCmd.Arg("uri", "URI of repository, e.g. s3://awesome-bucket/charts").
 		Required().
 		String()
+	initPublish := initCmd.Flag("publish", "The URI where the S3 bucket should be published").
+		Default("").
+		String()
 
 	pushCmd := cli.Command(actionPush, "Push chart to the repository.")
 	pushChartPath := pushCmd.Arg("chartPath", "Path to a chart, e.g. ./epicservice-0.5.1.tgz").
@@ -99,6 +102,9 @@ func main() {
 	reindexTargetRepository := reindexCmd.Arg("repo", "Target repository to reindex").
 		Required().
 		String()
+	reindexPublish := reindexCmd.Flag("publish", "The URI where the S3 bucket should be published").
+		Default("").
+		String()
 
 	deleteCmd := cli.Command(actionDelete, "Delete chart from the repository.").Alias("del")
 	deleteChartName := deleteCmd.Arg("chartName", "Name of chart to delete").
@@ -125,8 +131,9 @@ func main() {
 
 	case actionInit:
 		act = initAction{
-			uri: *initURI,
-			acl: *acl,
+			uri:        *initURI,
+			publishURI: *initPublish,
+			acl:        *acl,
 		}
 		defer fmt.Printf("Initialized empty repository at %s\n", *initURI)
 
@@ -143,8 +150,9 @@ func main() {
 
 	case actionReindex:
 		act = reindexAction{
-			repoName: *reindexTargetRepository,
-			acl:      *acl,
+			repoName:   *reindexTargetRepository,
+			publishURI: *reindexPublish,
+			acl:        *acl,
 		}
 		defer fmt.Printf("Repository %s was successfully reindexed.\n", *reindexTargetRepository)
 

cmd/helms3/push.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	"k8s.io/helm/pkg/chartutil"
@@ -80,7 +81,8 @@ func (act pushAction) Run(ctx context.Context) error {
 		return err
 	}
 
-	if cachedIndex, err := repo.LoadIndexFile(repoEntry.Cache); err == nil {
+	cachedIndex, err := repo.LoadIndexFile(repoEntry.Cache)
+	if err == nil {
 		// if cached index exists, check if the same chart version exists in it.
 		if cachedIndex.Has(chart.Metadata.Name, chart.Metadata.Version) {
 			if act.ignoreIfExists {
@@ -137,7 +139,8 @@ func (act pushAction) Run(ctx context.Context) error {
 
 	// Fetch current index, update it and upload it back.
 
-	b, err := storage.FetchRaw(ctx, repoEntry.URL+"/index.yaml")
+	indexURI := repoEntry.URL + "/index.yaml"
+	b, err := storage.FetchRaw(ctx, indexURI)
 	if err != nil {
 		return errors.WithMessage(err, "fetch current repo index")
 	}
@@ -147,7 +150,18 @@ func (act pushAction) Run(ctx context.Context) error {
 		return errors.WithMessage(err, "load index from downloaded file")
 	}
 
-	if err := idx.AddOrReplace(chart.GetMetadata(), fname, repoEntry.URL, hash); err != nil {
+	metadata, err := storage.GetMetadata(ctx, indexURI)
+	if err != nil {
+		return err
+	}
+
+	publishURI := metadata[strings.Title(awss3.MetaPublishURI)]
+	uri := repoEntry.URL
+	if publishURI != "" {
+		uri = publishURI
+	}
+
+	if err := idx.AddOrReplace(chart.GetMetadata(), fname, uri, hash); err != nil {
 		return errors.WithMessage(err, "add/replace chart in the index")
 	}
 	idx.SortEntries()
@@ -158,11 +172,17 @@ func (act pushAction) Run(ctx context.Context) error {
 	}
 
 	if !act.dryRun {
-		if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, idxReader); err != nil {
+		if err := storage.PutIndex(ctx, repoEntry.URL, publishURI, act.acl, idxReader); err != nil {
 			return errors.WithMessage(err, "upload index to s3")
 		}
 
-		if err := idx.WriteFile(repoEntry.Cache, 0644); err != nil {
+		localIndex := &index.Index{IndexFile: cachedIndex}
+		if err := localIndex.AddOrReplace(chart.GetMetadata(), fname, repoEntry.URL, hash); err != nil {
+			return errors.WithMessage(err, "add/replace chart in the index")
+		}
+		localIndex.SortEntries()
+
+		if err := localIndex.WriteFile(repoEntry.Cache, 0644); err != nil {
 			return errors.WithMessage(err, "update local index")
 		}
 	}

cmd/helms3/reindex.go

@@ -12,8 +12,9 @@ import (
 )
 
 type reindexAction struct {
-	repoName string
-	acl      string
+	repoName   string
+	publishURI string
+	acl        string
 }
 
 func (act reindexAction) Run(ctx context.Context) error {
@@ -30,11 +31,16 @@ func (act reindexAction) Run(ctx context.Context) error {
 
 	items, errs := storage.Traverse(ctx, repoEntry.URL)
 
+	uri := repoEntry.URL
+	if act.publishURI != "" {
+		uri = act.publishURI
+	}
+
 	builtIndex := make(chan *index.Index, 1)
 	go func() {
 		idx := index.New()
 		for item := range items {
-			idx.Add(item.Meta, item.Filename, repoEntry.URL, item.Hash)
+			idx.Add(item.Meta, item.Filename, uri, item.Hash)
 		}
 		idx.SortEntries()
 
@@ -52,7 +58,7 @@ func (act reindexAction) Run(ctx context.Context) error {
 		return errors.Wrap(err, "get index reader")
 	}
 
-	if err := storage.PutIndex(ctx, repoEntry.URL, act.acl, r); err != nil {
+	if err := storage.PutIndex(ctx, repoEntry.URL, act.publishURI, act.acl, r); err != nil {
 		return errors.Wrap(err, "upload index to the repository")
 	}
 

internal/awss3/storage.go

@@ -218,24 +218,34 @@ func (s *Storage) FetchRaw(ctx context.Context, uri string) ([]byte, error) {
 
 // Exists returns true if an object exists in the storage.
 func (s *Storage) Exists(ctx context.Context, uri string) (bool, error) {
+	if _, err := s.GetMetadata(ctx, uri); err != nil {
+		// That's weird that there is no NotFound constant in aws sdk.
+		if ae, ok := err.(awserr.Error); ok && ae.Code() == "NotFound" {
+			return false, nil
+		}
+		return false, errors.Wrap(err, "head s3 object")
+	}
+
+	return true, nil
+}
+
+// GetMetadata returns metadata associated with the object in storage
+func (s *Storage) GetMetadata(ctx context.Context, uri string) (map[string]string, error) {
 	bucket, key, err := parseURI(uri)
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
-	_, err = s3.New(s.session).HeadObject(&s3.HeadObjectInput{
+	result, err := s3.New(s.session).HeadObject(&s3.HeadObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
 	})
+
 	if err != nil {
-		// That's weird that there is no NotFound constant in aws sdk.
-		if ae, ok := err.(awserr.Error); ok && ae.Code() == "NotFound" {
-			return false, nil
-		}
-		return false, errors.Wrap(err, "head s3 object")
+		return nil, err
 	}
 
-	return true, nil
+	return aws.StringValueMap(result.Metadata), nil
 }
 
 // PutChart puts the chart file to the storage.
@@ -268,7 +278,7 @@ func (s *Storage) PutChart(ctx context.Context, uri string, r io.Reader, chartMe
 
 // PutIndex puts the index file to the storage.
 // uri must be in the form of s3 protocol: s3://bucket-name/key[...].
-func (s *Storage) PutIndex(ctx context.Context, uri string, acl string, r io.Reader) error {
+func (s *Storage) PutIndex(ctx context.Context, uri string, publishURI string, acl string, r io.Reader) error {
 	if strings.HasPrefix(uri, "index.yaml") {
 		return errors.New("uri must not contain \"index.yaml\" suffix, it appends automatically")
 	}
@@ -286,6 +296,9 @@ func (s *Storage) PutIndex(ctx context.Context, uri string, acl string, r io.Rea
 			ACL:                  aws.String(acl),
 			ServerSideEncryption: getSSE(),
 			Body:                 r,
+			Metadata: map[string]*string{
+				MetaPublishURI: aws.String(publishURI),
+			},
 		})
 	if err != nil {
 		return errors.Wrap(err, "upload index to S3 bucket")
@@ -339,4 +352,7 @@ const (
 
 	// metaChartDigest is a s3 object metadata key that represents chart digest.
 	metaChartDigest = "chart-digest"
+
+	// MetaPublishURI s3 object metadata key that stores the non-s3 URI to publish
+	MetaPublishURI = "helm-s3-publish-uri"
 )