diff --git a/context_test.go b/context_test.go index 2b95ebb87..8bdb42c1b 100644 --- a/context_test.go +++ b/context_test.go @@ -6,7 +6,9 @@ import ( "os" "testing" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/data" + "github.com/hairyhenderson/gomplate/v3/internal/datafs" "github.com/stretchr/testify/assert" ) @@ -30,6 +32,9 @@ func TestCreateContext(t *testing.T) { assert.NoError(t, err) assert.Empty(t, c) + fsmux := fsimpl.NewMux() + fsmux.Add(datafs.EnvFS) + fooURL := "env:///foo?type=application/yaml" barURL := "env:///bar?type=application/yaml" uf, _ := url.Parse(fooURL) @@ -39,6 +44,7 @@ func TestCreateContext(t *testing.T) { "foo": {URL: uf}, ".": {URL: ub}, }, + FSMux: fsmux, } os.Setenv("foo", "foo: bar") defer os.Unsetenv("foo") diff --git a/data/datasource.go b/data/datasource.go index a8ce5e742..5082b7899 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -2,21 +2,28 @@ package data import ( "context" + "encoding/json" "fmt" + "io/fs" + "io/ioutil" "mime" "net/http" "net/url" - "path/filepath" "sort" "strings" - "github.com/spf13/afero" - "github.com/pkg/errors" + "github.com/hairyhenderson/go-fsimpl" + "github.com/hairyhenderson/go-fsimpl/awssmfs" + "github.com/hairyhenderson/go-fsimpl/blobfs" + "github.com/hairyhenderson/go-fsimpl/filefs" + "github.com/hairyhenderson/go-fsimpl/gitfs" + "github.com/hairyhenderson/go-fsimpl/httpfs" + "github.com/hairyhenderson/go-fsimpl/vaultfs" "github.com/hairyhenderson/gomplate/v3/internal/config" + "github.com/hairyhenderson/gomplate/v3/internal/datafs" "github.com/hairyhenderson/gomplate/v3/libkv" - "github.com/hairyhenderson/gomplate/v3/vault" ) func regExtension(ext, typ string) { @@ -41,26 +48,11 @@ func (d *Data) registerReaders() { d.sourceReaders = make(map[string]func(context.Context, *Source, ...string) ([]byte, error)) d.sourceReaders["aws+smp"] = readAWSSMP - d.sourceReaders["aws+sm"] = readAWSSecretsManager d.sourceReaders["consul"] = readConsul d.sourceReaders["consul+http"] = readConsul d.sourceReaders["consul+https"] = readConsul - d.sourceReaders["env"] = readEnv - d.sourceReaders["file"] = readFile - d.sourceReaders["http"] = readHTTP - d.sourceReaders["https"] = readHTTP d.sourceReaders["merge"] = d.readMerge d.sourceReaders["stdin"] = readStdin - d.sourceReaders["vault"] = readVault - d.sourceReaders["vault+http"] = readVault - d.sourceReaders["vault+https"] = readVault - d.sourceReaders["s3"] = readBlob - d.sourceReaders["gs"] = readBlob - d.sourceReaders["git"] = readGit - d.sourceReaders["git+file"] = readGit - d.sourceReaders["git+http"] = readGit - d.sourceReaders["git+https"] = readGit - d.sourceReaders["git+ssh"] = readGit } // lookupReader - return the reader function for the given scheme @@ -83,10 +75,17 @@ type Data struct { Sources map[string]*Source sourceReaders map[string]func(context.Context, *Source, ...string) ([]byte, error) - cache map[string][]byte + cache map[string]*fileContent // headers from the --datasource-header/-H option that don't reference datasources from the commandline ExtraHeaders map[string]http.Header + + FSMux fsimpl.FSMux +} + +type fileContent struct { + b []byte + contentType string } // Cleanup - clean up datasources before shutting the process down - things @@ -140,91 +139,25 @@ func FromConfig(ctx context.Context, cfg *config.Config) *Data { // Source - a data source // Deprecated: will be replaced in future type Source struct { - Alias string - URL *url.URL - Header http.Header // used for http[s]: URLs, nil otherwise - fs afero.Fs // used for file: URLs, nil otherwise - hc *http.Client // used for http[s]: URLs, nil otherwise - vc *vault.Vault // used for vault: URLs, nil otherwise - kv *libkv.LibKV // used for consul:, etcd:, zookeeper: URLs, nil otherwise - asmpg awssmpGetter // used for aws+smp:, nil otherwise - awsSecretsManager awsSecretsManagerGetter // used for aws+sm, nil otherwise - mediaType string + Alias string + URL *url.URL + Header http.Header // used for http[s]: URLs, nil otherwise + kv *libkv.LibKV // used for consul:, etcd:, zookeeper: & boltdb: URLs, nil otherwise + asmpg awssmpGetter // used for aws+smp:, nil otherwise + mediaType string } func (s *Source) inherit(parent *Source) { - s.fs = parent.fs - s.hc = parent.hc - s.vc = parent.vc s.kv = parent.kv s.asmpg = parent.asmpg } func (s *Source) cleanup() { - if s.vc != nil { - s.vc.Logout() - } if s.kv != nil { s.kv.Logout() } } -// mimeType returns the MIME type to use as a hint for parsing the datasource. -// It's expected that the datasource will have already been read before -// this function is called, and so the Source's Type property may be already set. -// -// The MIME type is determined by these rules: -// 1. the 'type' URL query parameter is used if present -// 2. otherwise, the Type property on the Source is used, if present -// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered -// 4. otherwise, the default type of 'text/plain' is used -func (s *Source) mimeType(arg string) (mimeType string, err error) { - if len(arg) > 0 { - if strings.HasPrefix(arg, "//") { - arg = arg[1:] - } - if !strings.HasPrefix(arg, "/") { - arg = "/" + arg - } - } - argURL, err := url.Parse(arg) - if err != nil { - return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) - } - mediatype := argURL.Query().Get("type") - if mediatype == "" { - mediatype = s.URL.Query().Get("type") - } - - if mediatype == "" { - mediatype = s.mediaType - } - - // make it so + doesn't need to be escaped - mediatype = strings.ReplaceAll(mediatype, " ", "+") - - if mediatype == "" { - ext := filepath.Ext(argURL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype == "" { - ext := filepath.Ext(s.URL.Path) - mediatype = mime.TypeByExtension(ext) - } - - if mediatype != "" { - t, _, err := mime.ParseMediaType(mediatype) - if err != nil { - return "", errors.Wrapf(err, "MIME type was %q", mediatype) - } - mediatype = t - return mediatype, nil - } - - return textMimetype, nil -} - // String is the method to format the flag's value, part of the flag.Value interface. // The String method's output will be used in diagnostics. func (s *Source) String() string { @@ -281,41 +214,33 @@ func (d *Data) lookupSource(alias string) (*Source, error) { return source, nil } -func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (data, mimeType string, err error) { +func (d *Data) readDataSource(ctx context.Context, alias string, args ...string) (*fileContent, error) { source, err := d.lookupSource(alias) if err != nil { - return "", "", err + return nil, err } - b, err := d.readSource(ctx, source, args...) + fc, err := d.readSource(ctx, source, args...) if err != nil { - return "", "", errors.Wrapf(err, "Couldn't read datasource '%s'", alias) + return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", alias) } - subpath := "" - if len(args) > 0 { - subpath = args[0] - } - mimeType, err = source.mimeType(subpath) - if err != nil { - return "", "", err - } - return string(b), mimeType, nil + return fc, nil } // Include - func (d *Data) Include(alias string, args ...string) (string, error) { - data, _, err := d.readDataSource(d.Ctx, alias, args...) - return data, err + fc, err := d.readDataSource(d.Ctx, alias, args...) + return string(fc.b), err } // Datasource - func (d *Data) Datasource(alias string, args ...string) (interface{}, error) { - data, mimeType, err := d.readDataSource(d.Ctx, alias, args...) + fc, err := d.readDataSource(d.Ctx, alias, args...) if err != nil { return nil, err } - return parseData(mimeType, data) + return parseData(fc.contentType, string(fc.b)) } func parseData(mimeType, s string) (out interface{}, err error) { @@ -361,9 +286,9 @@ func (d *Data) DatasourceReachable(alias string, args ...string) bool { // readSource returns the (possibly cached) data from the given source, // as referenced by the given args -func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ([]byte, error) { +func (d *Data) readSource(ctx context.Context, source *Source, args ...string) (*fileContent, error) { if d.cache == nil { - d.cache = make(map[string][]byte) + d.cache = make(map[string]*fileContent) } cacheKey := source.Alias for _, v := range args { @@ -373,16 +298,107 @@ func (d *Data) readSource(ctx context.Context, source *Source, args ...string) ( if ok { return cached, nil } - r, err := d.lookupReader(source.URL.Scheme) + + var data []byte + + // TODO: initialize this elsewhere? + if d.FSMux == nil { + d.FSMux = fsimpl.NewMux() + d.FSMux.Add(filefs.FS) + d.FSMux.Add(httpfs.FS) + d.FSMux.Add(blobfs.FS) + d.FSMux.Add(gitfs.FS) + d.FSMux.Add(vaultfs.FS) + d.FSMux.Add(awssmfs.FS) + d.FSMux.Add(datafs.EnvFS) + } + + arg := "" + if len(args) > 0 { + arg = args[0] + } + + u, err := resolveURL(source.URL, arg) + if err != nil { + return nil, err + } + + // possible type hint + mimeType := u.Query().Get("type") + + u, fname := splitFSMuxURL(u) + + fsys, err := d.FSMux.Lookup(u.String()) + if err == nil { + f, err := fsys.Open(fname) + if err != nil { + return nil, fmt.Errorf("open (url: %q, name: %q): %w", u, fname, err) + } + + fi, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("stat (url: %q, name: %q): %w", u, fname, err) + } + + if mimeType == "" { + mimeType = fsimpl.ContentType(fi) + } + + if fi.IsDir() { + des, err := fs.ReadDir(fsys, fname) + if err != nil { + return nil, fmt.Errorf("readDir (url: %q, name: %s): %w", u, fname, err) + } + + entries := make([]string, len(des)) + for i, e := range des { + entries[i] = e.Name() + } + data, err = json.Marshal(entries) + if err != nil { + return nil, fmt.Errorf("json.Marshal: %w", err) + } + + mimeType = jsonArrayMimetype + } else { + data, err = ioutil.ReadAll(f) + + if err != nil { + return nil, fmt.Errorf("read (url: %q, name: %s): %w", u, fname, err) + } + } + + fc := &fileContent{data, mimeType} + d.cache[cacheKey] = fc + + return fc, nil + } + + // TODO: get rid of this, I guess? + r, err := d.lookupReader(u.Scheme) if err != nil { - return nil, errors.Wrap(err, "Datasource not yet supported") + return nil, fmt.Errorf("lookupReader (url: %q): %w", u, err) } - data, err := r(ctx, source, args...) + data, err = r(ctx, source, args...) if err != nil { return nil, err } - d.cache[cacheKey] = data - return data, nil + + if mimeType == "" { + subpath := "" + if len(args) > 0 { + subpath = args[0] + } + + mimeType, err = source.mimeType(subpath) + if err != nil { + return nil, err + } + } + + fc := &fileContent{data, mimeType} + d.cache[cacheKey] = fc + return fc, nil } // Show all datasources - @@ -394,3 +410,65 @@ func (d *Data) ListDatasources() []string { sort.Strings(datasources) return datasources } + +// resolveURL parses the relative URL rel against base, and returns the +// resolved URL. Differs from url.ResolveReference in that query parameters are +// added. In case of duplicates, params from rel are used. +func resolveURL(base *url.URL, rel string) (*url.URL, error) { + relURL, err := url.Parse(rel) + if err != nil { + return nil, err + } + + out := base.ResolveReference(relURL) + if base.RawQuery != "" { + bq := base.Query() + rq := relURL.Query() + for k := range rq { + bq.Set(k, rq.Get(k)) + } + out.RawQuery = bq.Encode() + } + + return out, nil +} + +// splitFSMuxURL splits a URL into a filesystem URL and a relative file path +func splitFSMuxURL(in *url.URL) (*url.URL, string) { + u := *in + + // base := path.Base(u.Path) + // if path.Dir(u.Path) == path.Clean(u.Path) { + // base = "." + // } + + base := strings.TrimPrefix(u.Path, "/") + + if base == "" && u.Opaque != "" { + base = u.Opaque + u.Opaque = "" + } + + if base == "" { + base = "." + } + + u.Path = "/" + + // handle some env-specific idiosyncrasies + // if u.Scheme == "env" { + // base = in.Path + // base = strings.TrimPrefix(base, "/") + // if base == "" { + // base = in.Opaque + // } + // } + // if u.Scheme == "vault" && !strings.HasSuffix(u.Path, "/") && u.Path != "" { + // u.Path += "/" + // } + // if u.Scheme == "s3" && !strings.HasSuffix(u.Path, "/") && u.Path != "" { + // u.Path += "/" + // } + + return &u, base +} diff --git a/data/datasource_aws_sm.go b/data/datasource_aws_sm.go deleted file mode 100644 index 8fce296ae..000000000 --- a/data/datasource_aws_sm.go +++ /dev/null @@ -1,87 +0,0 @@ -package data - -import ( - "context" - "fmt" - "net/url" - "path" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - - gaws "github.com/hairyhenderson/gomplate/v3/aws" -) - -// awsSecretsManagerGetter - A subset of Secrets Manager API for use in unit testing -type awsSecretsManagerGetter interface { - GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) -} - -func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { - if len(args) >= 2 { - err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", - sourceURL.Scheme, len(args)) - return nil, "", err - } - - p = sourceURL.Path - params = make(map[string]interface{}) - for key, val := range sourceURL.Query() { - params[key] = strings.Join(val, " ") - } - - if p == "" && sourceURL.Opaque != "" { - p = sourceURL.Opaque - } - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, "", err - } - - if parsed.Path != "" { - p = path.Join(p, parsed.Path) - if strings.HasSuffix(parsed.Path, "/") { - p += "/" - } - } - - for key, val := range parsed.Query() { - params[key] = strings.Join(val, " ") - } - } - return params, p, nil -} - -func readAWSSecretsManager(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if source.awsSecretsManager == nil { - source.awsSecretsManager = secretsmanager.New(gaws.SDKSession()) - } - - _, paramPath, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - return readAWSSecretsManagerParam(ctx, source, paramPath) -} - -func readAWSSecretsManagerParam(ctx context.Context, source *Source, paramPath string) ([]byte, error) { - input := &secretsmanager.GetSecretValueInput{ - SecretId: aws.String(paramPath), - } - - response, err := source.awsSecretsManager.GetSecretValueWithContext(ctx, input) - if err != nil { - return nil, fmt.Errorf("reading aws+sm source %q: %w", source.Alias, err) - } - - if response.SecretString != nil { - return []byte(*response.SecretString), nil - } - - return response.SecretBinary, nil -} diff --git a/data/datasource_aws_sm_test.go b/data/datasource_aws_sm_test.go deleted file mode 100644 index c274dcf8c..000000000 --- a/data/datasource_aws_sm_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/secretsmanager" - "github.com/stretchr/testify/assert" -) - -// DummyAWSSecretsManagerSecretGetter - test double -type DummyAWSSecretsManagerSecretGetter struct { - t *testing.T - secretValut *secretsmanager.GetSecretValueOutput - err awserr.Error - mockGetSecretValue func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) -} - -func (d DummyAWSSecretsManagerSecretGetter) GetSecretValueWithContext(ctx context.Context, input *secretsmanager.GetSecretValueInput, opts ...request.Option) (*secretsmanager.GetSecretValueOutput, error) { - if d.mockGetSecretValue != nil { - output, err := d.mockGetSecretValue(input) - return output, err - } - if d.err != nil { - return nil, d.err - } - assert.NotNil(d.t, d.secretValut, "Must provide a param if no error!") - return d.secretValut, nil -} - -func simpleAWSSecretsManagerSourceHelper(dummyGetter awsSecretsManagerGetter) *Source { - return &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "aws+sm", - Path: "/foo", - }, - awsSecretsManager: dummyGetter, - } -} - -func TestAWSSecretsManager_ParseAWSSecretsManagerArgs(t *testing.T) { - _, _, err := parseDatasourceURLArgs(mustParseURL("base"), "extra", "too many!") - assert.Error(t, err) - - tplain := map[string]interface{}{"type": "text/plain"} - - data := []struct { - eParams map[string]interface{} - u string - ePath string - args string - }{ - {u: "noddy", ePath: "noddy"}, - {u: "base", ePath: "base/extra", args: "extra"}, - {u: "/foo/", ePath: "/foo/extra", args: "/extra"}, - {u: "aws+sm:///foo", ePath: "/foo/bar", args: "bar"}, - {u: "aws+sm:foo", ePath: "foo"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar"}, - {u: "aws+sm:foo", ePath: "foo/baz", args: "baz"}, - {u: "aws+sm:foo/bar", ePath: "foo/bar/baz", args: "baz"}, - {u: "aws+sm:/foo/bar", ePath: "/foo/bar/baz", args: "baz"}, - {u: "aws+sm:///foo", ePath: "/foo/dir/", args: "dir/"}, - {u: "aws+sm:///foo/", ePath: "/foo/"}, - {u: "aws+sm:///foo/", ePath: "/foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo?type=text/plain", ePath: "foo/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:foo/bar?type=text/plain", ePath: "foo/bar/baz", args: "baz"}, - {eParams: tplain, u: "aws+sm:/foo/bar?type=text/plain", ePath: "/foo/bar/baz", args: "baz"}, - { - eParams: map[string]interface{}{ - "type": "application/json", - "param": "quux", - }, - u: "aws+sm:/foo/bar?type=text/plain", - ePath: "/foo/bar/baz/qux", - args: "baz/qux?type=application/json¶m=quux", - }, - } - - for _, d := range data { - args := []string{d.args} - if d.args == "" { - args = nil - } - params, p, err := parseDatasourceURLArgs(mustParseURL(d.u), args...) - assert.NoError(t, err) - if d.eParams == nil { - assert.Empty(t, params) - } else { - assert.EqualValues(t, d.eParams, params) - } - assert.Equal(t, d.ePath, p) - } -} - -func TestAWSSecretsManager_GetParameterSetup(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar") - assert.True(t, calledOk) - assert.Nil(t, err) -} - -func TestAWSSecretsManager_GetParameterSetupWrongArgs(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "/bar", "/foo", "/bla") - assert.False(t, calledOk) - assert.Error(t, err) -} - -func TestAWSSecretsManager_GetParameterMissing(t *testing.T) { - expectedErr := awserr.New("ParameterNotFound", "Test of error message", nil) - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - err: expectedErr, - }) - - _, err := readAWSSecretsManager(context.Background(), s, "") - assert.Error(t, err, "Test of error message") -} - -func TestAWSSecretsManager_ReadSecret(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretString: aws.String("blub")}, nil - }, - }) - - output, err := readAWSSecretsManagerParam(context.Background(), s, "/foo/bar") - assert.True(t, calledOk) - assert.NoError(t, err) - assert.Equal(t, []byte("blub"), output) -} - -func TestAWSSecretsManager_ReadSecretBinary(t *testing.T) { - calledOk := false - s := simpleAWSSecretsManagerSourceHelper(DummyAWSSecretsManagerSecretGetter{ - t: t, - mockGetSecretValue: func(input *secretsmanager.GetSecretValueInput) (*secretsmanager.GetSecretValueOutput, error) { - assert.Equal(t, "/foo/bar", *input.SecretId) - calledOk = true - return &secretsmanager.GetSecretValueOutput{SecretBinary: []byte("supersecret")}, nil - }, - }) - - output, err := readAWSSecretsManagerParam(context.Background(), s, "/foo/bar") - assert.True(t, calledOk) - assert.NoError(t, err) - assert.Equal(t, []byte("supersecret"), output) -} diff --git a/data/datasource_awssmp.go b/data/datasource_awssmp.go index d61affe82..3f9a14e5f 100644 --- a/data/datasource_awssmp.go +++ b/data/datasource_awssmp.go @@ -2,6 +2,9 @@ package data import ( "context" + "fmt" + "net/url" + "path" "strings" "github.com/aws/aws-sdk-go/aws" @@ -76,3 +79,40 @@ func listAWSSMPParams(ctx context.Context, source *Source, paramPath string) ([] output, err := ToJSON(listing) return []byte(output), err } + +func parseDatasourceURLArgs(sourceURL *url.URL, args ...string) (params map[string]interface{}, p string, err error) { + if len(args) >= 2 { + err = fmt.Errorf("maximum two arguments to %s datasource: alias, extraPath (found %d)", + sourceURL.Scheme, len(args)) + return nil, "", err + } + + p = sourceURL.Path + params = make(map[string]interface{}) + for key, val := range sourceURL.Query() { + params[key] = strings.Join(val, " ") + } + + if p == "" && sourceURL.Opaque != "" { + p = sourceURL.Opaque + } + + if len(args) == 1 { + parsed, err := url.Parse(args[0]) + if err != nil { + return nil, "", err + } + + if parsed.Path != "" { + p = path.Join(p, parsed.Path) + if strings.HasSuffix(parsed.Path, "/") { + p += "/" + } + } + + for key, val := range parsed.Query() { + params[key] = strings.Join(val, " ") + } + } + return params, p, nil +} diff --git a/data/datasource_blob.go b/data/datasource_blob.go deleted file mode 100644 index bb316e7f3..000000000 --- a/data/datasource_blob.go +++ /dev/null @@ -1,170 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "io" - "mime" - "net/url" - "path" - "strings" - - gaws "github.com/hairyhenderson/gomplate/v3/aws" - "github.com/hairyhenderson/gomplate/v3/env" - "github.com/pkg/errors" - - "gocloud.dev/blob" - "gocloud.dev/blob/gcsblob" - "gocloud.dev/blob/s3blob" - "gocloud.dev/gcp" -) - -func readBlob(ctx context.Context, source *Source, args ...string) (output []byte, err error) { - if len(args) >= 2 { - return nil, errors.New("maximum two arguments to blob datasource: alias, extraPath") - } - - key := source.URL.Path - if len(args) == 1 { - key = path.Join(key, args[0]) - } - - opener, err := newOpener(ctx, source.URL) - if err != nil { - return nil, err - } - - mux := blob.URLMux{} - mux.RegisterBucket(source.URL.Scheme, opener) - - u := blobURL(source.URL) - bucket, err := mux.OpenBucket(ctx, u) - if err != nil { - return nil, err - } - defer bucket.Close() - - var r func(context.Context, *blob.Bucket, string) (string, []byte, error) - if strings.HasSuffix(key, "/") { - r = listBucket - } else { - r = getBlob - } - - mediaType, data, err := r(ctx, bucket, key) - if mediaType != "" { - source.mediaType = mediaType - } - return data, err -} - -// create the correct kind of blob.BucketURLOpener for the given URL -func newOpener(ctx context.Context, u *url.URL) (opener blob.BucketURLOpener, err error) { - switch u.Scheme { - case "s3": - // set up a "regular" gomplate AWS SDK session - sess := gaws.SDKSession() - // see https://gocloud.dev/concepts/urls/#muxes - opener = &s3blob.URLOpener{ConfigProvider: sess} - case "gs": - creds, err := gcp.DefaultCredentials(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve GCP credentials") - } - - client, err := gcp.NewHTTPClient( - gcp.DefaultTransport(), - gcp.CredentialsTokenSource(creds)) - if err != nil { - return nil, errors.Wrap(err, "failed to create GCP HTTP client") - } - opener = &gcsblob.URLOpener{ - Client: client, - } - } - return opener, nil -} - -func getBlob(ctx context.Context, bucket *blob.Bucket, key string) (mediaType string, data []byte, err error) { - key = strings.TrimPrefix(key, "/") - attr, err := bucket.Attributes(ctx, key) - if err != nil { - return "", nil, errors.Wrapf(err, "failed to retrieve attributes for %s", key) - } - if attr.ContentType != "" { - mt, _, e := mime.ParseMediaType(attr.ContentType) - if e != nil { - return "", nil, e - } - mediaType = mt - } - data, err = bucket.ReadAll(ctx, key) - return mediaType, data, errors.Wrapf(err, "failed to read %s", key) -} - -// calls the bucket listing API, returning a JSON Array -func listBucket(ctx context.Context, bucket *blob.Bucket, path string) (mediaType string, data []byte, err error) { - path = strings.TrimPrefix(path, "/") - opts := &blob.ListOptions{ - Prefix: path, - Delimiter: "/", - } - li := bucket.List(opts) - keys := []string{} - for { - obj, err := li.Next(ctx) - if err == io.EOF { - break - } - if err != nil { - return "", nil, err - } - keys = append(keys, strings.TrimPrefix(obj.Key, path)) - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(keys); err != nil { - return "", nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - data = b[:len(b)-1] - return jsonArrayMimetype, data, nil -} - -// copy/sanitize the URL for the Go CDK - it doesn't like params it can't parse -func blobURL(u *url.URL) string { - out := cloneURL(u) - q := out.Query() - - for param := range q { - switch u.Scheme { - case "s3": - switch param { - case "region", "endpoint", "disableSSL", "s3ForcePathStyle": - default: - q.Del(param) - } - case "gs": - switch param { - case "access_id", "private_key_path": - default: - q.Del(param) - } - } - } - - if u.Scheme == "s3" { - // handle AWS_S3_ENDPOINT env var - endpoint := env.Getenv("AWS_S3_ENDPOINT") - if endpoint != "" { - q.Set("endpoint", endpoint) - } - } - - out.RawQuery = q.Encode() - - return out.String() -} diff --git a/data/datasource_blob_test.go b/data/datasource_blob_test.go deleted file mode 100644 index 7763d8e91..000000000 --- a/data/datasource_blob_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package data - -import ( - "bytes" - "context" - "net/http/httptest" - "net/url" - "os" - "testing" - - "github.com/johannesboyne/gofakes3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - - "github.com/stretchr/testify/assert" -) - -func setupTestBucket(t *testing.T) (*httptest.Server, *url.URL) { - backend := s3mem.New() - faker := gofakes3.New(backend) - ts := httptest.NewServer(faker.Server()) - - err := backend.CreateBucket("mybucket") - assert.NoError(t, err) - c := "hello" - err = putFile(backend, "mybucket", "file1", "text/plain", c) - assert.NoError(t, err) - - c = `{"value": "goodbye world"}` - err = putFile(backend, "mybucket", "file2", "application/json", c) - assert.NoError(t, err) - - c = `value: what a world` - err = putFile(backend, "mybucket", "file3", "application/yaml", c) - assert.NoError(t, err) - - c = `value: out of this world` - err = putFile(backend, "mybucket", "dir1/file1", "application/yaml", c) - assert.NoError(t, err) - - c = `value: foo` - err = putFile(backend, "mybucket", "dir1/file2", "application/yaml", c) - assert.NoError(t, err) - - u, _ := url.Parse(ts.URL) - return ts, u -} - -func putFile(backend gofakes3.Backend, bucket, file, mime, content string) error { - _, err := backend.PutObject( - bucket, - file, - map[string]string{"Content-Type": mime}, - bytes.NewBufferString(content), - int64(len(content)), - ) - return err -} - -func TestReadBlob(t *testing.T) { - _, err := readBlob(context.Background(), nil, "foo", "bar") - assert.Error(t, err) - - ts, u := setupTestBucket(t) - defer ts.Close() - - os.Setenv("AWS_ANON", "true") - defer os.Unsetenv("AWS_ANON") - - d, err := NewData([]string{"-d", "data=s3://mybucket/file1?region=us-east-1&disableSSL=true&s3ForcePathStyle=true&type=text/plain&endpoint=" + u.Host}, nil) - assert.NoError(t, err) - - var expected interface{} - expected = "hello" - out, err := d.Datasource("data") - assert.NoError(t, err) - assert.Equal(t, expected, out) - - os.Unsetenv("AWS_ANON") - - os.Setenv("AWS_ACCESS_KEY_ID", "fake") - os.Setenv("AWS_SECRET_ACCESS_KEY", "fake") - defer os.Unsetenv("AWS_ACCESS_KEY_ID") - defer os.Unsetenv("AWS_SECRET_ACCESS_KEY") - os.Setenv("AWS_S3_ENDPOINT", u.Host) - defer os.Unsetenv("AWS_S3_ENDPOINT") - - d, err = NewData([]string{"-d", "data=s3://mybucket/file2?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = map[string]interface{}{"value": "goodbye world"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.Equal(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = []interface{}{"dir1/", "file1", "file2", "file3"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.EqualValues(t, expected, out) - - d, err = NewData([]string{"-d", "data=s3://mybucket/dir1/?region=us-east-1&disableSSL=true&s3ForcePathStyle=true"}, nil) - assert.NoError(t, err) - - expected = []interface{}{"file1", "file2"} - out, err = d.Datasource("data") - assert.NoError(t, err) - assert.EqualValues(t, expected, out) -} - -func TestBlobURL(t *testing.T) { - data := []struct { - in string - expected string - }{ - {"s3://foo/bar/baz", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?type=hello/world", "s3://foo/bar/baz"}, - {"s3://foo/bar/baz?region=us-east-1", "s3://foo/bar/baz?region=us-east-1"}, - {"s3://foo/bar/baz?disableSSL=true&type=text/csv", "s3://foo/bar/baz?disableSSL=true"}, - {"s3://foo/bar/baz?type=text/csv&s3ForcePathStyle=true&endpoint=1.2.3.4", "s3://foo/bar/baz?endpoint=1.2.3.4&s3ForcePathStyle=true"}, - {"gs://foo/bar/baz", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?type=foo/bar", "gs://foo/bar/baz"}, - {"gs://foo/bar/baz?access_id=123", "gs://foo/bar/baz?access_id=123"}, - {"gs://foo/bar/baz?private_key_path=/foo/bar", "gs://foo/bar/baz?private_key_path=%2Ffoo%2Fbar"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar", "gs://foo/bar/baz?private_key_path=key.json"}, - {"gs://foo/bar/baz?private_key_path=key.json&foo=bar&access_id=abcd", "gs://foo/bar/baz?access_id=abcd&private_key_path=key.json"}, - } - - for _, d := range data { - u, _ := url.Parse(d.in) - out := blobURL(u) - assert.Equal(t, d.expected, out) - } -} diff --git a/data/datasource_env.go b/data/datasource_env.go deleted file mode 100644 index f1e2e5aad..000000000 --- a/data/datasource_env.go +++ /dev/null @@ -1,19 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/hairyhenderson/gomplate/v3/env" -) - -func readEnv(ctx context.Context, source *Source, args ...string) (b []byte, err error) { - n := source.URL.Path - n = strings.TrimPrefix(n, "/") - if n == "" { - n = source.URL.Opaque - } - - b = []byte(env.Getenv(n)) - return b, nil -} diff --git a/data/datasource_env_test.go b/data/datasource_env_test.go deleted file mode 100644 index c7723b3b2..000000000 --- a/data/datasource_env_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package data - -import ( - "context" - "net/url" - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func mustParseURL(in string) *url.URL { - u, _ := url.Parse(in) - return u -} - -func TestReadEnv(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - os.Setenv("HELLO_WORLD", "hello world") - defer os.Unsetenv("HELLO_WORLD") - os.Setenv("HELLO_UNIVERSE", "hello universe") - defer os.Unsetenv("HELLO_UNIVERSE") - - source := &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD")} - - actual, err := readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:/HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "foo", URL: mustParseURL("env:///HELLO_WORLD?foo=bar")} - - actual, err = readEnv(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) -} diff --git a/data/datasource_file.go b/data/datasource_file.go deleted file mode 100644 index 7619f0399..000000000 --- a/data/datasource_file.go +++ /dev/null @@ -1,85 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" - - "github.com/pkg/errors" -) - -func readFile(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.fs == nil { - source.fs = afero.NewOsFs() - } - - p := filepath.FromSlash(source.URL.Path) - - if len(args) == 1 { - parsed, err := url.Parse(args[0]) - if err != nil { - return nil, err - } - - if parsed.Path != "" { - p = filepath.Join(p, parsed.Path) - } - - // reset the media type - it may have been set by a parent dir read - source.mediaType = "" - } - - // make sure we can access the file - i, err := source.fs.Stat(p) - if err != nil { - return nil, errors.Wrapf(err, "Can't stat %s", p) - } - - if strings.HasSuffix(p, string(filepath.Separator)) { - source.mediaType = jsonArrayMimetype - if i.IsDir() { - return readFileDir(source, p) - } - return nil, errors.Errorf("%s is not a directory", p) - } - - f, err := source.fs.OpenFile(p, os.O_RDONLY, 0) - if err != nil { - return nil, errors.Wrapf(err, "Can't open %s", p) - } - - defer f.Close() - - b, err := ioutil.ReadAll(f) - if err != nil { - return nil, errors.Wrapf(err, "Can't read %s", p) - } - return b, nil -} - -func readFileDir(source *Source, p string) ([]byte, error) { - names, err := afero.ReadDir(source.fs, p) - if err != nil { - return nil, err - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} diff --git a/data/datasource_file_test.go b/data/datasource_file_test.go deleted file mode 100644 index 8f2a7f0a2..000000000 --- a/data/datasource_file_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package data - -import ( - "context" - "testing" - - "github.com/spf13/afero" - - "github.com/stretchr/testify/assert" -) - -func TestReadFile(t *testing.T) { - ctx := context.Background() - - content := []byte(`hello world`) - fs := afero.NewMemMapFs() - - _ = fs.Mkdir("/tmp", 0777) - f, _ := fs.Create("/tmp/foo") - _, _ = f.Write(content) - - _ = fs.Mkdir("/tmp/partial", 0777) - f, _ = fs.Create("/tmp/partial/foo.txt") - _, _ = f.Write(content) - _, _ = fs.Create("/tmp/partial/bar.txt") - _, _ = fs.Create("/tmp/partial/baz.txt") - _ = f.Close() - - source := &Source{Alias: "foo", URL: mustParseURL("file:///tmp/foo")} - source.fs = fs - - actual, err := readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "bogus", URL: mustParseURL("file:///bogus")} - source.fs = fs - _, err = readFile(ctx, source) - assert.Error(t, err) - - source = &Source{Alias: "partial", URL: mustParseURL("file:///tmp/partial")} - source.fs = fs - actual, err = readFile(ctx, source, "foo.txt") - assert.NoError(t, err) - assert.Equal(t, content, actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/")} - source.fs = fs - actual, err = readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fs - actual, err = readFile(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(`["bar.txt","baz.txt","foo.txt"]`), actual) - mime, err := source.mimeType("") - assert.NoError(t, err) - assert.Equal(t, "application/json", mime) - - source = &Source{Alias: "dir", URL: mustParseURL("file:///tmp/partial/?type=application/json")} - source.fs = fs - actual, err = readFile(ctx, source, "foo.txt") - assert.NoError(t, err) - assert.Equal(t, content, actual) - mime, err = source.mimeType("") - assert.NoError(t, err) - assert.Equal(t, "application/json", mime) -} diff --git a/data/datasource_git.go b/data/datasource_git.go deleted file mode 100644 index c2673dd3c..000000000 --- a/data/datasource_git.go +++ /dev/null @@ -1,328 +0,0 @@ -package data - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/url" - "os" - "path" - "path/filepath" - "strings" - - "github.com/hairyhenderson/gomplate/v3/base64" - "github.com/hairyhenderson/gomplate/v3/env" - "github.com/rs/zerolog" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/memory" -) - -func readGit(ctx context.Context, source *Source, args ...string) ([]byte, error) { - g := gitsource{} - - u := source.URL - repoURL, path, err := g.parseGitPath(u, args...) - if err != nil { - return nil, err - } - - depth := 1 - if u.Scheme == "git+file" { - // we can't do shallow clones for filesystem repos apparently - depth = 0 - } - - fs, _, err := g.clone(ctx, repoURL, depth) - if err != nil { - return nil, err - } - - mimeType, out, err := g.read(fs, path) - if mimeType != "" { - source.mediaType = mimeType - } - return out, err -} - -type gitsource struct { -} - -func (g gitsource) parseArgURL(arg string) (u *url.URL, err error) { - if strings.HasPrefix(arg, "//") { - u, err = url.Parse(arg[1:]) - u.Path = "/" + u.Path - } else { - u, err = url.Parse(arg) - } - - if err != nil { - return nil, fmt.Errorf("failed to parse arg %s: %w", arg, err) - } - return u, err -} - -func (g gitsource) parseQuery(orig, arg *url.URL) string { - q := orig.Query() - pq := arg.Query() - for k, vs := range pq { - for _, v := range vs { - q.Add(k, v) - } - } - return q.Encode() -} - -func (g gitsource) parseArgPath(u *url.URL, arg string) (repo, p string) { - // if the source URL already specified a repo and subpath, the whole - // arg is interpreted as subpath - if strings.Contains(u.Path, "//") || strings.HasPrefix(arg, "//") { - return "", arg - } - - parts := strings.SplitN(arg, "//", 2) - repo = parts[0] - if len(parts) == 2 { - p = "/" + parts[1] - } - return repo, p -} - -// Massage the URL and args together to produce the repo to clone, -// and the path to read. -// The path is delimited from the repo by '//' -func (g gitsource) parseGitPath(u *url.URL, args ...string) (out *url.URL, p string, err error) { - if u == nil { - return nil, "", fmt.Errorf("parseGitPath: no url provided (%v)", u) - } - // copy the input url so we can modify it - out = cloneURL(u) - - parts := strings.SplitN(out.Path, "//", 2) - switch len(parts) { - case 1: - p = "/" - case 2: - p = "/" + parts[1] - - i := strings.LastIndex(out.Path, p) - out.Path = out.Path[:i-1] - } - - if len(args) > 0 { - argURL, uerr := g.parseArgURL(args[0]) - if uerr != nil { - return nil, "", uerr - } - repo, argpath := g.parseArgPath(u, argURL.Path) - out.Path = path.Join(out.Path, repo) - p = path.Join(p, argpath) - - out.RawQuery = g.parseQuery(u, argURL) - - if argURL.Fragment != "" { - out.Fragment = argURL.Fragment - } - } - return out, p, err -} - -//nolint: interfacer -func cloneURL(u *url.URL) *url.URL { - out, _ := url.Parse(u.String()) - return out -} - -// refFromURL - extract the ref from the URL fragment if present -func (g gitsource) refFromURL(u *url.URL) plumbing.ReferenceName { - switch { - case strings.HasPrefix(u.Fragment, "refs/"): - return plumbing.ReferenceName(u.Fragment) - case u.Fragment != "": - return plumbing.NewBranchReferenceName(u.Fragment) - default: - return plumbing.ReferenceName("") - } -} - -// refFromRemoteHead - extract the ref from the remote HEAD, to work around -// hard-coded 'master' default branch in go-git. -// Should be unnecessary once https://github.com/go-git/go-git/issues/249 is -// fixed. -func (g gitsource) refFromRemoteHead(ctx context.Context, u *url.URL, auth transport.AuthMethod) (plumbing.ReferenceName, error) { - e, err := transport.NewEndpoint(u.String()) - if err != nil { - return "", err - } - - cli, err := client.NewClient(e) - if err != nil { - return "", err - } - - s, err := cli.NewUploadPackSession(e, auth) - if err != nil { - return "", err - } - - info, err := s.AdvertisedReferencesContext(ctx) - if err != nil { - return "", err - } - - refs, err := info.AllReferences() - if err != nil { - return "", err - } - - headRef, ok := refs["HEAD"] - if !ok { - return "", fmt.Errorf("no HEAD ref found") - } - - return headRef.Target(), nil -} - -// clone a repo for later reading through http(s), git, or ssh. u must be the URL to the repo -// itself, and must have any file path stripped -func (g gitsource) clone(ctx context.Context, repoURL *url.URL, depth int) (billy.Filesystem, *git.Repository, error) { - fs := memfs.New() - storer := memory.NewStorage() - - // preserve repoURL by cloning it - u := cloneURL(repoURL) - - auth, err := g.auth(u) - if err != nil { - return nil, nil, err - } - - if strings.HasPrefix(u.Scheme, "git+") { - scheme := u.Scheme[len("git+"):] - u.Scheme = scheme - } - - ref := g.refFromURL(u) - u.Fragment = "" - u.RawQuery = "" - - // attempt to get the ref from the remote so we don't default to master - if ref == "" { - ref, err = g.refFromRemoteHead(ctx, u, auth) - if err != nil { - zerolog.Ctx(ctx).Warn(). - Stringer("repoURL", u). - Err(err). - Msg("failed to get ref from remote, using default") - } - } - - opts := &git.CloneOptions{ - URL: u.String(), - Auth: auth, - Depth: depth, - ReferenceName: ref, - SingleBranch: true, - Tags: git.NoTags, - } - repo, err := git.CloneContext(ctx, storer, fs, opts) - if u.Scheme == "file" && err == transport.ErrRepositoryNotFound && !strings.HasSuffix(u.Path, ".git") { - // maybe this has a `.git` subdirectory... - u = cloneURL(repoURL) - u.Path = path.Join(u.Path, ".git") - return g.clone(ctx, u, depth) - } - if err != nil { - return nil, nil, fmt.Errorf("git clone for %v failed: %w", repoURL, err) - } - return fs, repo, nil -} - -// read - reads the provided path out of a git repo -func (g gitsource) read(fs billy.Filesystem, path string) (string, []byte, error) { - fi, err := fs.Stat(path) - if err != nil { - return "", nil, fmt.Errorf("can't stat %s: %w", path, err) - } - if fi.IsDir() || strings.HasSuffix(path, string(filepath.Separator)) { - out, rerr := g.readDir(fs, path) - return jsonArrayMimetype, out, rerr - } - - f, err := fs.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return "", nil, fmt.Errorf("can't open %s: %w", path, err) - } - - b, err := ioutil.ReadAll(f) - if err != nil { - return "", nil, fmt.Errorf("can't read %s: %w", path, err) - } - - return "", b, nil -} - -func (g gitsource) readDir(fs billy.Filesystem, path string) ([]byte, error) { - names, err := fs.ReadDir(path) - if err != nil { - return nil, fmt.Errorf("couldn't read dir %s: %w", path, err) - } - files := make([]string, len(names)) - for i, v := range names { - files[i] = v.Name() - } - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(files); err != nil { - return nil, err - } - b := buf.Bytes() - // chop off the newline added by the json encoder - return b[:len(b)-1], nil -} - -/* -auth methods: -- ssh named key (no password support) - - GIT_SSH_KEY (base64-encoded) or GIT_SSH_KEY_FILE (base64-encoded, or not) -- ssh agent auth (preferred) -- http basic auth (for github, gitlab, bitbucket tokens) -- http token auth (bearer token, somewhat unusual) -*/ -func (g gitsource) auth(u *url.URL) (auth transport.AuthMethod, err error) { - user := u.User.Username() - switch u.Scheme { - case "git+http", "git+https": - if pass, ok := u.User.Password(); ok { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if pass := env.Getenv("GIT_HTTP_PASSWORD"); pass != "" { - auth = &http.BasicAuth{Username: user, Password: pass} - } else if tok := env.Getenv("GIT_HTTP_TOKEN"); tok != "" { - // note docs on TokenAuth - this is rarely to be used - auth = &http.TokenAuth{Token: tok} - } - case "git+ssh": - k := env.Getenv("GIT_SSH_KEY") - if k != "" { - var key []byte - key, err = base64.Decode(k) - if err != nil { - key = []byte(k) - } - auth, err = ssh.NewPublicKeys(user, key, "") - } else { - auth, err = ssh.NewSSHAgentAuth(user) - } - } - return auth, err -} diff --git a/data/datasource_git_test.go b/data/datasource_git_test.go deleted file mode 100644 index 0c20c5583..000000000 --- a/data/datasource_git_test.go +++ /dev/null @@ -1,486 +0,0 @@ -package data - -import ( - "context" - "encoding/base64" - "fmt" - "io/ioutil" - "net/url" - "os" - "strings" - "testing" - "time" - - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-billy/v5/osfs" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/server" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/go-git/go-git/v5/storage/filesystem" - - "golang.org/x/crypto/ssh/testdata" - - "gotest.tools/v3/assert" - is "gotest.tools/v3/assert/cmp" -) - -func TestParseArgPath(t *testing.T) { - t.Parallel() - g := gitsource{} - - data := []struct { - url string - arg string - repo, path string - }{ - {"git+file:///foo//foo", - "/bar", - "", "/bar"}, - {"git+file:///foo//bar", - "/baz//qux", - "", "/baz//qux"}, - {"git+https://example.com/foo", - "/bar", - "/bar", ""}, - {"git+https://example.com/foo", - "//bar", - "", "//bar"}, - {"git+https://example.com/foo//bar", - "//baz", - "", "//baz"}, - {"git+https://example.com/foo", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo?type=t", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo#master", - "/bar//baz", - "/bar", "/baz"}, - {"git+https://example.com/foo", - "//bar", - "", "//bar"}, - {"git+https://example.com/foo?type=t", - "//baz", - "", "//baz"}, - {"git+https://example.com/foo?type=t#v1", - "//bar", - "", "//bar"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.arg, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - repo, path := g.parseArgPath(u, d.arg) - assert.Equal(t, d.repo, repo) - assert.Equal(t, d.path, path) - }) - } -} - -func TestParseGitPath(t *testing.T) { - t.Parallel() - g := gitsource{} - _, _, err := g.parseGitPath(nil) - assert.ErrorContains(t, err, "") - - u := mustParseURL("http://example.com//foo") - assert.Equal(t, "//foo", u.Path) - parts := strings.SplitN(u.Path, "//", 2) - assert.Equal(t, 2, len(parts)) - assert.DeepEqual(t, []string{"", "foo"}, parts) - - data := []struct { - url string - args string - repo, path string - }{ - {"git+https://github.com/hairyhenderson/gomplate//docs-src/content/functions/aws.yml", - "", - "git+https://github.com/hairyhenderson/gomplate", - "/docs-src/content/functions/aws.yml"}, - {"git+ssh://github.com/hairyhenderson/gomplate.git", - "", - "git+ssh://github.com/hairyhenderson/gomplate.git", - "/"}, - {"https://github.com", - "", - "https://github.com", - "/"}, - {"git://example.com/foo//file.txt#someref", - "", - "git://example.com/foo#someref", "/file.txt"}, - {"git+file:///home/foo/repo//file.txt#someref", - "", - "git+file:///home/foo/repo#someref", "/file.txt"}, - {"git+file:///repo", - "", - "git+file:///repo", "/"}, - {"git+file:///foo//foo", - "", - "git+file:///foo", "/foo"}, - {"git+file:///foo//foo", - "/bar", - "git+file:///foo", "/foo/bar"}, - {"git+file:///foo//bar", - // in this case the // is meaningless - "/baz//qux", - "git+file:///foo", "/bar/baz/qux"}, - {"git+https://example.com/foo", - "/bar", - "git+https://example.com/foo/bar", "/"}, - {"git+https://example.com/foo", - "//bar", - "git+https://example.com/foo", "/bar"}, - {"git+https://example.com//foo", - "/bar", - "git+https://example.com", "/foo/bar"}, - {"git+https://example.com/foo//bar", - "//baz", - "git+https://example.com/foo", "/bar/baz"}, - {"git+https://example.com/foo", - "/bar//baz", - "git+https://example.com/foo/bar", "/baz"}, - {"git+https://example.com/foo?type=t", - "/bar//baz", - "git+https://example.com/foo/bar?type=t", "/baz"}, - {"git+https://example.com/foo#master", - "/bar//baz", - "git+https://example.com/foo/bar#master", "/baz"}, - {"git+https://example.com/foo", - "/bar//baz?type=t", - "git+https://example.com/foo/bar?type=t", "/baz"}, - {"git+https://example.com/foo", - "/bar//baz#master", - "git+https://example.com/foo/bar#master", "/baz"}, - {"git+https://example.com/foo", - "//bar?type=t", - "git+https://example.com/foo?type=t", "/bar"}, - {"git+https://example.com/foo", - "//bar#master", - "git+https://example.com/foo#master", "/bar"}, - {"git+https://example.com/foo?type=t", - "//bar#master", - "git+https://example.com/foo?type=t#master", "/bar"}, - {"git+https://example.com/foo?type=t#v1", - "//bar?type=j#v2", - "git+https://example.com/foo?type=t&type=j#v2", "/bar"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:(%q,%q)==(%q,%q)", i, d.url, d.args, d.repo, d.path), func(t *testing.T) { - t.Parallel() - u, _ := url.Parse(d.url) - args := []string{d.args} - if d.args == "" { - args = nil - } - repo, path, err := g.parseGitPath(u, args...) - assert.NilError(t, err) - assert.Equal(t, d.repo, repo.String()) - assert.Equal(t, d.path, path) - }) - } -} - -func TestReadGitRepo(t *testing.T) { - g := gitsource{} - fs := setupGitRepo(t) - fs, err := fs.Chroot("/repo") - assert.NilError(t, err) - - _, _, err = g.read(fs, "/bogus") - assert.ErrorContains(t, err, "can't stat /bogus") - - mtype, out, err := g.read(fs, "/foo") - assert.NilError(t, err) - assert.Equal(t, `["bar"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar") - assert.NilError(t, err) - assert.Equal(t, `["hi.txt"]`, string(out)) - assert.Equal(t, jsonArrayMimetype, mtype) - - mtype, out, err = g.read(fs, "/foo/bar/hi.txt") - assert.NilError(t, err) - assert.Equal(t, `hello world`, string(out)) - assert.Equal(t, "", mtype) -} - -var testHashes = map[string]string{} - -func setupGitRepo(t *testing.T) billy.Filesystem { - fs := memfs.New() - fs.MkdirAll("/repo/.git", os.ModeDir) - repo, _ := fs.Chroot("/repo") - dot, _ := repo.Chroot("/.git") - s := filesystem.NewStorage(dot, cache.NewObjectLRUDefault()) - - r, err := git.Init(s, repo) - assert.NilError(t, err) - - // default to main - h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) - err = s.SetReference(h) - assert.NilError(t, err) - - // config needs to be created after setting up a "normal" fs repo - // this is possibly a bug in git-go? - c, err := r.Config() - assert.NilError(t, err) - - c.Init.DefaultBranch = "main" - - s.SetConfig(c) - assert.NilError(t, err) - - w, err := r.Worktree() - assert.NilError(t, err) - - repo.MkdirAll("/foo/bar", os.ModeDir) - f, err := repo.Create("/foo/bar/hi.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("hello world")) - assert.NilError(t, err) - _, err = w.Add(f.Name()) - assert.NilError(t, err) - hash, err := w.Commit("initial commit", &git.CommitOptions{Author: &object.Signature{}}) - assert.NilError(t, err) - - ref, err := r.CreateTag("v1", hash, nil) - assert.NilError(t, err) - testHashes["v1"] = hash.String() - - branchName := plumbing.NewBranchReferenceName("mybranch") - err = w.Checkout(&git.CheckoutOptions{ - Branch: branchName, - Hash: ref.Hash(), - Create: true, - }) - assert.NilError(t, err) - - f, err = repo.Create("/secondfile.txt") - assert.NilError(t, err) - _, err = f.Write([]byte("another file\n")) - assert.NilError(t, err) - n := f.Name() - _, err = w.Add(n) - assert.NilError(t, err) - hash, err = w.Commit("second commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - }, - }) - ref = plumbing.NewHashReference(branchName, hash) - assert.NilError(t, err) - testHashes["mybranch"] = ref.Hash().String() - - // make the repo dirty - _, err = f.Write([]byte("dirty file")) - assert.NilError(t, err) - - // set up a bare repo - fs.MkdirAll("/bare.git", os.ModeDir) - fs.MkdirAll("/barewt", os.ModeDir) - repo, _ = fs.Chroot("/barewt") - dot, _ = fs.Chroot("/bare.git") - s = filesystem.NewStorage(dot, nil) - - r, err = git.Init(s, repo) - assert.NilError(t, err) - - w, err = r.Worktree() - assert.NilError(t, err) - - f, err = repo.Create("/hello.txt") - assert.NilError(t, err) - f.Write([]byte("hello world")) - w.Add(f.Name()) - _, err = w.Commit("initial commit", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@doe.org", - When: time.Now(), - }, - }) - assert.NilError(t, err) - - return fs -} - -func overrideFSLoader(fs billy.Filesystem) { - l := server.NewFilesystemLoader(fs) - client.InstallProtocol("file", server.NewClient(l)) -} - -func TestOpenFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fs, _, err := g.clone(ctx, mustParseURL("git+file:///repo"), 0) - assert.NilError(t, err) - - f, err := fs.Open("/foo/bar/hi.txt") - assert.NilError(t, err) - b, _ := ioutil.ReadAll(f) - assert.Equal(t, "hello world", string(b)) - - _, repo, err := g.clone(ctx, mustParseURL("git+file:///repo#main"), 0) - assert.NilError(t, err) - - ref, err := repo.Reference(plumbing.NewBranchReferenceName("main"), true) - assert.NilError(t, err) - assert.Equal(t, "refs/heads/main", ref.Name().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo#refs/tags/v1"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, testHashes["v1"], ref.Hash().String()) - - _, repo, err = g.clone(ctx, mustParseURL("git+file:///repo/#mybranch"), 0) - assert.NilError(t, err) - - ref, err = repo.Head() - assert.NilError(t, err) - assert.Equal(t, "refs/heads/mybranch", ref.Name().String()) - assert.Equal(t, testHashes["mybranch"], ref.Hash().String()) -} - -func TestOpenBareFileRepo(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - g := gitsource{} - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - fs, _, err := g.clone(ctx, mustParseURL("git+file:///bare.git"), 0) - assert.NilError(t, err) - - f, err := fs.Open("/hello.txt") - assert.NilError(t, err) - b, _ := ioutil.ReadAll(f) - assert.Equal(t, "hello world", string(b)) -} - -func TestReadGit(t *testing.T) { - ctx := context.Background() - repoFS := setupGitRepo(t) - - overrideFSLoader(repoFS) - defer overrideFSLoader(osfs.New("")) - - s := &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git//hello.txt"), - } - b, err := readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "hello world", string(b)) - - s = &Source{ - Alias: "hi", - URL: mustParseURL("git+file:///bare.git"), - } - b, err = readGit(ctx, s) - assert.NilError(t, err) - assert.Equal(t, "application/array+json", s.mediaType) - assert.Equal(t, `["hello.txt"]`, string(b)) -} - -func TestGitAuth(t *testing.T) { - g := gitsource{} - a, err := g.auth(mustParseURL("git+file:///bare.git")) - assert.NilError(t, err) - assert.Equal(t, nil, a) - - a, err = g.auth(mustParseURL("git+https://example.com/foo")) - assert.NilError(t, err) - assert.Assert(t, is.Nil(a)) - - a, err = g.auth(mustParseURL("git+https://user:swordfish@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - - os.Setenv("GIT_HTTP_PASSWORD", "swordfish") - defer os.Unsetenv("GIT_HTTP_PASSWORD") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.BasicAuth{Username: "user", Password: "swordfish"}, a) - os.Unsetenv("GIT_HTTP_PASSWORD") - - os.Setenv("GIT_HTTP_TOKEN", "mytoken") - defer os.Unsetenv("GIT_HTTP_TOKEN") - a, err = g.auth(mustParseURL("git+https://user@example.com/foo")) - assert.NilError(t, err) - assert.DeepEqual(t, &http.TokenAuth{Token: "mytoken"}, a) - os.Unsetenv("GIT_HTTP_TOKEN") - - if os.Getenv("SSH_AUTH_SOCK") == "" { - t.Log("no SSH_AUTH_SOCK - skipping ssh agent test") - } else { - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - sa, ok := a.(*ssh.PublicKeysCallback) - assert.Equal(t, true, ok) - assert.Equal(t, "git", sa.User) - } - - key := string(testdata.PEMBytes["ed25519"]) - os.Setenv("GIT_SSH_KEY", key) - defer os.Unsetenv("GIT_SSH_KEY") - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok := a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") - - key = base64.StdEncoding.EncodeToString(testdata.PEMBytes["ed25519"]) - os.Setenv("GIT_SSH_KEY", key) - defer os.Unsetenv("GIT_SSH_KEY") - a, err = g.auth(mustParseURL("git+ssh://git@example.com/foo")) - assert.NilError(t, err) - ka, ok = a.(*ssh.PublicKeys) - assert.Equal(t, true, ok) - assert.Equal(t, "git", ka.User) - os.Unsetenv("GIT_SSH_KEY") -} - -func TestRefFromURL(t *testing.T) { - t.Parallel() - g := gitsource{} - data := []struct { - url, expected string - }{ - {"git://localhost:1234/foo/bar.git//baz", ""}, - {"git+http://localhost:1234/foo/bar.git//baz", ""}, - {"git+ssh://localhost:1234/foo/bar.git//baz", ""}, - {"git+file:///foo/bar.git//baz", ""}, - {"git://localhost:1234/foo/bar.git//baz#master", "refs/heads/master"}, - {"git+http://localhost:1234/foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - {"git+ssh://localhost:1234/foo/bar.git//baz#refs/tags/foo", "refs/tags/foo"}, - {"git+file:///foo/bar.git//baz#mybranch", "refs/heads/mybranch"}, - } - - for _, d := range data { - out := g.refFromURL(mustParseURL(d.url)) - assert.Equal(t, plumbing.ReferenceName(d.expected), out) - } -} diff --git a/data/datasource_http.go b/data/datasource_http.go deleted file mode 100644 index 03e1ac4c0..000000000 --- a/data/datasource_http.go +++ /dev/null @@ -1,63 +0,0 @@ -package data - -import ( - "context" - "io/ioutil" - "mime" - "net/http" - "net/url" - "time" - - "github.com/pkg/errors" -) - -func buildURL(base *url.URL, args ...string) (*url.URL, error) { - if len(args) == 0 { - return base, nil - } - p, err := url.Parse(args[0]) - if err != nil { - return nil, errors.Wrapf(err, "bad sub-path %s", args[0]) - } - return base.ResolveReference(p), nil -} - -func readHTTP(ctx context.Context, source *Source, args ...string) ([]byte, error) { - if source.hc == nil { - source.hc = &http.Client{Timeout: time.Second * 5} - } - u, err := buildURL(source.URL, args...) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - if err != nil { - return nil, err - } - req.Header = source.Header - res, err := source.hc.Do(req) - if err != nil { - return nil, err - } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - err = res.Body.Close() - if err != nil { - return nil, err - } - if res.StatusCode != 200 { - err := errors.Errorf("Unexpected HTTP status %d on GET from %s: %s", res.StatusCode, source.URL, string(body)) - return nil, err - } - ctypeHdr := res.Header.Get("Content-Type") - if ctypeHdr != "" { - mediatype, _, e := mime.ParseMediaType(ctypeHdr) - if e != nil { - return nil, e - } - source.mediaType = mediatype - } - return body, nil -} diff --git a/data/datasource_http_test.go b/data/datasource_http_test.go deleted file mode 100644 index 5d2797128..000000000 --- a/data/datasource_http_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package data - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func must(r interface{}, err error) interface{} { - if err != nil { - panic(err) - } - return r -} - -func setupHTTP(code int, mimetype string, body string) (*httptest.Server, *http.Client) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - - w.Header().Set("Content-Type", mimetype) - w.WriteHeader(code) - if body == "" { - // mirror back the headers - fmt.Fprintln(w, must(marshalObj(r.Header, json.Marshal))) - } else { - fmt.Fprintln(w, body) - } - })) - - client := &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - return url.Parse(server.URL) - }, - }, - } - - return server, client -} - -func TestHTTPFile(t *testing.T) { - server, client := setupHTTP(200, "application/json; charset=utf-8", `{"hello": "world"}`) - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - - expected := map[string]interface{}{ - "hello": "world", - } - - actual, err := data.Datasource("foo") - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - actual, err = data.Datasource(server.URL) - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestHTTPFileWithHeaders(t *testing.T) { - server, client := setupHTTP(200, jsonMimetype, "") - defer server.Close() - - sources := make(map[string]*Source) - sources["foo"] = &Source{ - Alias: "foo", - URL: &url.URL{ - Scheme: "http", - Host: "example.com", - Path: "/foo", - }, - hc: client, - Header: http.Header{ - "Foo": {"bar"}, - "foo": {"baz"}, - "User-Agent": {}, - "Accept-Encoding": {"test"}, - }, - } - data := &Data{ - Ctx: context.Background(), - Sources: sources, - } - expected := http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - } - actual, err := data.Datasource("foo") - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) - - expected = http.Header{ - "Accept-Encoding": {"test"}, - "Foo": {"bar", "baz"}, - "User-Agent": {"Go-http-client/1.1"}, - } - data = &Data{ - Ctx: context.Background(), - Sources: sources, - ExtraHeaders: map[string]http.Header{server.URL: expected}, - } - actual, err = data.Datasource(server.URL) - assert.NoError(t, err) - assert.Equal(t, must(marshalObj(expected, json.Marshal)), must(marshalObj(actual, json.Marshal))) -} - -func TestBuildURL(t *testing.T) { - expected := "https://example.com/index.html" - base := mustParseURL(expected) - u, err := buildURL(base) - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/index.html" - base = mustParseURL("https://example.com") - u, err = buildURL(base, "index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/a/b/c/index.html" - base = mustParseURL("https://example.com/a/") - u, err = buildURL(base, "b/c/index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) - - expected = "https://example.com/bar/baz/index.html" - base = mustParseURL("https://example.com/foo") - u, err = buildURL(base, "bar/baz/index.html") - assert.NoError(t, err) - assert.Equal(t, expected, u.String()) -} diff --git a/data/datasource_merge.go b/data/datasource_merge.go index 136a3779b..b0beaca19 100644 --- a/data/datasource_merge.go +++ b/data/datasource_merge.go @@ -2,8 +2,12 @@ package data import ( "context" + "fmt" + "io/fs" + "path" "strings" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/coll" "github.com/hairyhenderson/gomplate/v3/internal/config" @@ -24,7 +28,7 @@ func (d *Data) readMerge(ctx context.Context, source *Source, args ...string) ([ opaque := source.URL.Opaque parts := strings.Split(opaque, "|") if len(parts) < 2 { - return nil, errors.New("need at least 2 datasources to merge") + return nil, fmt.Errorf("need at least 2 datasources to merge") } data := make([]map[string]interface{}, len(parts)) for i, part := range parts { @@ -43,16 +47,32 @@ func (d *Data) readMerge(ctx context.Context, source *Source, args ...string) ([ } subSource.inherit(source) - b, err := d.readSource(ctx, subSource) + u := *subSource.URL + + base := path.Base(u.Path) + if base == "/" { + base = "." + } + + u.Path = path.Dir(u.Path) + + fsys, err := d.FSMux.Lookup(u.String()) if err != nil { - return nil, errors.Wrapf(err, "Couldn't read datasource '%s'", part) + return nil, fmt.Errorf("lookup %s: %w", u.String(), err) } - mimeType, err := subSource.mimeType("") + b, err := fs.ReadFile(fsys, base) if err != nil { - return nil, errors.Wrapf(err, "failed to read datasource %s", subSource.URL) + return nil, fmt.Errorf("readFile (fs: %q, name: %q): %w", &u, base, err) } + fi, err := fs.Stat(fsys, base) + if err != nil { + return nil, fmt.Errorf("stat (fs: %q, name: %q): %w", &u, base, err) + } + + mimeType := fsimpl.ContentType(fi) + data[i], err = parseMap(mimeType, string(b)) if err != nil { return nil, err diff --git a/data/datasource_merge_test.go b/data/datasource_merge_test.go index 2d81c98a1..b951253cb 100644 --- a/data/datasource_merge_test.go +++ b/data/datasource_merge_test.go @@ -2,13 +2,15 @@ package data import ( "context" + "io/fs" "net/url" "os" + "path" "path/filepath" "testing" + "testing/fstest" - "github.com/spf13/afero" - + "github.com/hairyhenderson/go-fsimpl" "github.com/stretchr/testify/assert" ) @@ -21,31 +23,27 @@ func TestReadMerge(t *testing.T) { mergedContent := "goodnight: moon\nhello: world\n" - fs := afero.NewMemMapFs() - - _ = fs.Mkdir("/tmp", 0777) - f, _ := fs.Create("/tmp/jsonfile.json") - _, _ = f.WriteString(jsonContent) - f, _ = fs.Create("/tmp/array.json") - _, _ = f.WriteString(arrayContent) - f, _ = fs.Create("/tmp/yamlfile.yaml") - _, _ = f.WriteString(yamlContent) - f, _ = fs.Create("/tmp/textfile.txt") - _, _ = f.WriteString(`plain text...`) + fsys := fstest.MapFS{} + fsys["tmp"] = &fstest.MapFile{Mode: fs.ModeDir | 0777} + fsys["tmp/jsonfile.json"] = &fstest.MapFile{Data: []byte(jsonContent)} + fsys["tmp/array.json"] = &fstest.MapFile{Data: []byte(arrayContent)} + fsys["tmp/yamlfile.yaml"] = &fstest.MapFile{Data: []byte(yamlContent)} + fsys["tmp/textfile.txt"] = &fstest.MapFile{Data: []byte(`plain text...`)} + // workding dir with volume name trimmed wd, _ := os.Getwd() - _ = fs.Mkdir(wd, 0777) - f, _ = fs.Create(filepath.Join(wd, "jsonfile.json")) - _, _ = f.WriteString(jsonContent) - f, _ = fs.Create(filepath.Join(wd, "array.json")) - _, _ = f.WriteString(arrayContent) - f, _ = fs.Create(filepath.Join(wd, "yamlfile.yaml")) - _, _ = f.WriteString(yamlContent) - f, _ = fs.Create(filepath.Join(wd, "textfile.txt")) - _, _ = f.WriteString(`plain text...`) + vol := filepath.VolumeName(wd) + wd = wd[len(vol)+1:] + + fsys[path.Join(wd, "jsonfile.json")] = &fstest.MapFile{Data: []byte(jsonContent)} + fsys[path.Join(wd, "array.json")] = &fstest.MapFile{Data: []byte(arrayContent)} + fsys[path.Join(wd, "yamlfile.yaml")] = &fstest.MapFile{Data: []byte(yamlContent)} + fsys[path.Join(wd, "textfile.txt")] = &fstest.MapFile{Data: []byte(`plain text...`)} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(&fsys, "file")) source := &Source{Alias: "foo", URL: mustParseURL("merge:file:///tmp/jsonfile.json|file:///tmp/yamlfile.yaml")} - source.fs = fs d := &Data{ Sources: map[string]*Source{ "foo": source, @@ -56,6 +54,7 @@ func TestReadMerge(t *testing.T) { "badtype": {Alias: "badtype", URL: mustParseURL("file:///tmp/textfile.txt?type=foo/bar")}, "array": {Alias: "array", URL: mustParseURL("file:///tmp/array.json?type=" + url.QueryEscape(jsonArrayMimetype))}, }, + FSMux: fsmux, } actual, err := d.readMerge(ctx, source) diff --git a/data/datasource_test.go b/data/datasource_test.go index 5badec534..fa97d3ce9 100644 --- a/data/datasource_test.go +++ b/data/datasource_test.go @@ -3,18 +3,22 @@ package data import ( "context" "fmt" + "io/fs" "net/http" "net/url" - "runtime" "testing" + "testing/fstest" + "github.com/hairyhenderson/go-fsimpl" "github.com/hairyhenderson/gomplate/v3/internal/config" - "github.com/spf13/afero" "github.com/stretchr/testify/assert" ) -const osWindows = "windows" +func mustParseURL(in string) *url.URL { + u, _ := url.Parse(in) + return u +} func TestNewData(t *testing.T) { d, err := NewData(nil, nil) @@ -42,55 +46,50 @@ func TestNewData(t *testing.T) { } func TestDatasource(t *testing.T) { - setup := func(ext, mime string, contents []byte) *Data { + setup := func(t *testing.T, ext string, contents []byte) *Data { + t.Helper() fname := "foo." + ext - fs := afero.NewMemMapFs() - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write(contents) + + fsys := fstest.MapFS{} + fsys["tmp"] = &fstest.MapFile{Mode: fs.ModeDir | 0777} + fsys["tmp/"+fname] = &fstest.MapFile{Data: contents} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { - Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, - mediaType: mime, - fs: fs, + Alias: "foo", + URL: mustParseURL("file:///tmp/" + fname), }, } - return &Data{Sources: sources} + return &Data{Sources: sources, FSMux: fsmux} } - test := func(ext, mime string, contents []byte, expected interface{}) { - data := setup(ext, mime, contents) - actual, err := data.Datasource("foo") + test := func(t *testing.T, ext, mime string, contents []byte, expected interface{}) { + t.Helper() + data := setup(t, ext, contents) + + actual, err := data.Datasource("foo", "?type="+mime) assert.NoError(t, err) assert.Equal(t, expected, actual) } - testObj := func(ext, mime string, contents []byte) { - test(ext, mime, contents, + testObj := func(t *testing.T, ext, mime string, contents []byte) { + test(t, ext, mime, contents, map[string]interface{}{ "hello": map[string]interface{}{"cruel": "world"}, }) } - testObj("json", jsonMimetype, []byte(`{"hello":{"cruel":"world"}}`)) - testObj("yml", yamlMimetype, []byte("hello:\n cruel: world\n")) - test("json", jsonMimetype, []byte(`[1, "two", true]`), + testObj(t, "json", jsonMimetype, []byte(`{"hello":{"cruel":"world"}}`)) + testObj(t, "yml", yamlMimetype, []byte("hello:\n cruel: world\n")) + test(t, "json", jsonMimetype, []byte(`[1, "two", true]`), []interface{}{1, "two", true}) - test("yaml", yamlMimetype, []byte("---\n- 1\n- two\n- true\n"), + test(t, "yaml", yamlMimetype, []byte("---\n- 1\n- two\n- true\n"), []interface{}{1, "two", true}) - d := setup("", textMimetype, nil) + d := setup(t, "", nil) actual, err := d.Datasource("foo") assert.NoError(t, err) assert.Equal(t, "", actual) @@ -100,35 +99,24 @@ func TestDatasource(t *testing.T) { } func TestDatasourceReachable(t *testing.T) { - fname := "foo.json" - fs := afero.NewMemMapFs() - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write([]byte("{}")) + fsys := fstest.MapFS{} + fsys["tmp/foo.json"] = &fstest.MapFile{Data: []byte("{}")} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, + URL: mustParseURL("file:///tmp/foo.json"), mediaType: jsonMimetype, - fs: fs, }, "bar": { Alias: "bar", URL: &url.URL{Scheme: "file", Path: "/bogus"}, - fs: fs, }, } - data := &Data{Sources: sources} + data := &Data{Sources: sources, FSMux: fsmux} assert.True(t, data.DatasourceReachable("foo")) assert.False(t, data.DatasourceReachable("bar")) @@ -144,35 +132,22 @@ func TestDatasourceExists(t *testing.T) { } func TestInclude(t *testing.T) { - ext := "txt" contents := "hello world" - fname := "foo." + ext - fs := afero.NewMemMapFs() - - var uPath string - var f afero.File - if runtime.GOOS == osWindows { - _ = fs.Mkdir("C:\\tmp", 0777) - f, _ = fs.Create("C:\\tmp\\" + fname) - uPath = "C:/tmp/" + fname - } else { - _ = fs.Mkdir("/tmp", 0777) - f, _ = fs.Create("/tmp/" + fname) - uPath = "/tmp/" + fname - } - _, _ = f.Write([]byte(contents)) + + fsys := fstest.MapFS{} + fsys["tmp/foo.txt"] = &fstest.MapFile{Data: []byte(contents)} + + fsmux := fsimpl.NewMux() + fsmux.Add(fsimpl.WrappedFSProvider(fsys, "file")) sources := map[string]*Source{ "foo": { Alias: "foo", - URL: &url.URL{Scheme: "file", Path: uPath}, + URL: mustParseURL("file:///tmp/foo.txt"), mediaType: textMimetype, - fs: fs, }, } - data := &Data{ - Sources: sources, - } + data := &Data{Sources: sources, FSMux: fsmux} actual, err := data.Include("foo") assert.NoError(t, err) assert.Equal(t, contents, actual) @@ -245,103 +220,6 @@ func TestDefineDatasource(t *testing.T) { assert.Equal(t, "application/x-env", m) } -func TestMimeType(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} - _, err := s.mimeType("") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - expected string - }{ - {"http://example.com/foo.json", - "", - jsonMimetype}, - {"http://example.com/foo.json", - "text/foo", - "text/foo"}, - {"http://example.com/foo.json?type=application/yaml", - "text/foo", - "application/yaml"}, - {"http://example.com/list?type=application/array%2Bjson", - "text/foo", - "application/array+json"}, - {"http://example.com/list?type=application/array+json", - "", - "application/array+json"}, - {"http://example.com/unknown", - "", - "text/plain"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType("") - assert.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - -func TestMimeTypeWithArg(t *testing.T) { - s := &Source{URL: mustParseURL("http://example.com")} - _, err := s.mimeType("h\nttp://foo") - assert.Error(t, err) - - data := []struct { - url string - mediaType string - arg string - expected string - }{ - {"http://example.com/unknown", - "", - "/foo.json", - "application/json"}, - {"http://example.com/unknown", - "", - "foo.json", - "application/json"}, - {"http://example.com/", - "text/foo", - "/foo.json", - "text/foo"}, - {"git+https://example.com/myrepo", - "", - "//foo.yaml", - "application/yaml"}, - {"http://example.com/foo.json", - "", - "/foo.yaml", - "application/yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml", - "application/array+yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "", - "/foo.yaml?type=application/yaml", - "application/yaml"}, - {"http://example.com/foo.json?type=application/array+yaml", - "text/plain", - "/foo.yaml?type=application/yaml", - "application/yaml"}, - } - - for i, d := range data { - d := d - t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { - s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} - mt, err := s.mimeType(d.arg) - assert.NoError(t, err) - assert.Equal(t, d.expected, mt) - }) - } -} - func TestFromConfig(t *testing.T) { ctx := context.Background() @@ -426,3 +304,79 @@ func TestListDatasources(t *testing.T) { assert.Equal(t, []string{"bar", "foo"}, data.ListDatasources()) } + +func TestSplitFSMuxURL(t *testing.T) { + t.Skip() + testdata := []struct { + in string + arg string + url string + file string + }{ + {"http://example.com/foo.json", "", "http://example.com/", "foo.json"}, + { + "http://example.com/foo.json?type=application/array+yaml", + "", + "http://example.com/?type=application/array+yaml", + "foo.json", + }, + { + "vault:///secret/a/b/c", "", + "vault:///", + "secret/a/b/c", + }, + { + "vault:///secret/a/b/", "", + "vault:///", + "secret/a/b", + }, + { + "s3://bucket/a/b/", "", + "s3://bucket/", + "a/b", + }, + { + "vault:///", "foo/bar", + "vault:///", + "foo/bar", + }, + { + "consul://myhost/foo/?q=1", "bar/baz", + "consul://myhost/?q=1", + "foo/bar/baz", + }, + { + "consul://myhost/foo/?q=1", "bar/baz", + "consul://myhost/?q=1", + "foo/bar/baz", + }, + { + "git+https://example.com/myrepo", "//foo.yaml", + "git+https://example.com/myrepo", "foo.yaml", + }, + { + "ssh://git@github.com/hairyhenderson/go-which.git//a/b/", + "c/d?q=1", + "ssh://git@github.com/hairyhenderson/go-which.git?q=1", + "a/b/c/d", + }, + } + + for _, d := range testdata { + u, err := url.Parse(d.in) + assert.NoError(t, err) + url, file := splitFSMuxURL(u) + assert.Equal(t, d.url, url.String()) + assert.Equal(t, d.file, file) + } +} + +func TestResolveURL(t *testing.T) { + out, err := resolveURL(mustParseURL("http://example.com/foo.json"), "bar.json") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/bar.json", out.String()) + + out, err = resolveURL(mustParseURL("http://example.com/a/b/?n=2"), "bar.json?q=1") + assert.NoError(t, err) + assert.Equal(t, "http://example.com/a/b/bar.json?n=2&q=1", out.String()) +} diff --git a/data/datasource_vault.go b/data/datasource_vault.go deleted file mode 100644 index 09f6dcb17..000000000 --- a/data/datasource_vault.go +++ /dev/null @@ -1,48 +0,0 @@ -package data - -import ( - "context" - "strings" - - "github.com/pkg/errors" - - "github.com/hairyhenderson/gomplate/v3/vault" -) - -func readVault(ctx context.Context, source *Source, args ...string) (data []byte, err error) { - if source.vc == nil { - source.vc, err = vault.New(source.URL) - if err != nil { - return nil, err - } - err = source.vc.Login() - if err != nil { - return nil, err - } - } - - params, p, err := parseDatasourceURLArgs(source.URL, args...) - if err != nil { - return nil, err - } - - source.mediaType = jsonMimetype - switch { - case len(params) > 0: - data, err = source.vc.Write(p, params) - case strings.HasSuffix(p, "/"): - source.mediaType = jsonArrayMimetype - data, err = source.vc.List(p) - default: - data, err = source.vc.Read(p) - } - if err != nil { - return nil, err - } - - if len(data) == 0 { - return nil, errors.Errorf("no value found for path %s", p) - } - - return data, nil -} diff --git a/data/datasource_vault_test.go b/data/datasource_vault_test.go deleted file mode 100644 index 8b470abc6..000000000 --- a/data/datasource_vault_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package data - -import ( - "context" - "net/url" - "testing" - - "github.com/hairyhenderson/gomplate/v3/vault" - "github.com/stretchr/testify/assert" -) - -func TestReadVault(t *testing.T) { - ctx := context.Background() - - expected := "{\"value\":\"foo\"}\n" - server, v := vault.MockServer(200, `{"data":`+expected+`}`) - defer server.Close() - - source := &Source{ - Alias: "foo", - URL: &url.URL{Scheme: "vault", Path: "/secret/foo"}, - mediaType: textMimetype, - vc: v, - } - - r, err := readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "bar") - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - r, err = readVault(ctx, source, "?param=value") - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - source.URL, _ = url.Parse("vault:///secret/foo?param1=value1¶m2=value2") - r, err = readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) - - expected = "[\"one\",\"two\"]\n" - server, source.vc = vault.MockServer(200, `{"data":{"keys":`+expected+`}}`) - defer server.Close() - source.URL, _ = url.Parse("vault:///secret/foo/") - r, err = readVault(ctx, source) - assert.NoError(t, err) - assert.Equal(t, []byte(expected), r) -} diff --git a/data/mimetypes.go b/data/mimetypes.go index bdc12ad4c..3db302cc1 100644 --- a/data/mimetypes.go +++ b/data/mimetypes.go @@ -1,5 +1,15 @@ package data +import ( + "fmt" + "mime" + "net/url" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + const ( textMimetype = "text/plain" csvMimetype = "text/csv" @@ -23,3 +33,115 @@ func mimeAlias(m string) string { } return m } + +// mimeType returns the MIME type to use as a hint for parsing the datasource. +// It's expected that the datasource will have already been read before +// this function is called, and so the Source's Type property may be already set. +// +// The MIME type is determined by these rules: +// 1. the 'type' URL query parameter is used if present +// 2. otherwise, the Type property on the Source is used, if present +// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered +// 4. otherwise, the default type of 'text/plain' is used +func (s *Source) mimeType(arg string) (mimeType string, err error) { + if len(arg) > 0 { + if strings.HasPrefix(arg, "//") { + arg = arg[1:] + } + if !strings.HasPrefix(arg, "/") { + arg = "/" + arg + } + } + argURL, err := url.Parse(arg) + if err != nil { + return "", fmt.Errorf("mimeType: couldn't parse arg %q: %w", arg, err) + } + mediatype := argURL.Query().Get("type") + if mediatype == "" { + mediatype = s.URL.Query().Get("type") + } + + if mediatype == "" { + mediatype = s.mediaType + } + + // make it so + doesn't need to be escaped + mediatype = strings.ReplaceAll(mediatype, " ", "+") + + if mediatype == "" { + ext := filepath.Ext(argURL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype == "" { + ext := filepath.Ext(s.URL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype != "" { + t, _, err := mime.ParseMediaType(mediatype) + if err != nil { + return "", errors.Wrapf(err, "MIME type was %q", mediatype) + } + mediatype = t + return mediatype, nil + } + + return textMimetype, nil +} + +// mimeType returns the MIME type to use as a hint for parsing the datasource. +// It's expected that the datasource will have already been read before +// this function is called, and so the Source's Type property may be already set. +// +// The MIME type is determined by these rules: +// 1. the 'type' URL query parameter is used if present +// 2. otherwise, the Type property on the Source is used, if present +// 3. otherwise, a MIME type is calculated from the file extension, if the extension is registered +// 4. otherwise, the default type of 'text/plain' is used +func guessMimeType(base *url.URL, name, mimeGuess string) (mimeType string, err error) { + if len(name) > 0 { + if strings.HasPrefix(name, "//") { + name = name[1:] + } + if !strings.HasPrefix(name, "/") { + name = "/" + name + } + } + nameURL, err := url.Parse(name) + if err != nil { + return "", fmt.Errorf("mimeType: couldn't parse name %q: %w", name, err) + } + mediatype := nameURL.Query().Get("type") + if mediatype == "" { + mediatype = base.Query().Get("type") + } + + if mediatype == "" { + mediatype = mimeGuess + } + + // make it so + doesn't need to be escaped + mediatype = strings.ReplaceAll(mediatype, " ", "+") + + if mediatype == "" { + ext := filepath.Ext(nameURL.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype == "" { + ext := filepath.Ext(base.Path) + mediatype = mime.TypeByExtension(ext) + } + + if mediatype != "" { + t, _, err := mime.ParseMediaType(mediatype) + if err != nil { + return "", errors.Wrapf(err, "MIME type was %q", mediatype) + } + mediatype = t + return mediatype, nil + } + + return textMimetype, nil +} diff --git a/data/mimetypes_test.go b/data/mimetypes_test.go index 0dd1ab052..8f36dc06d 100644 --- a/data/mimetypes_test.go +++ b/data/mimetypes_test.go @@ -1,9 +1,10 @@ package data import ( + "fmt" "testing" - "gotest.tools/v3/assert" + "github.com/stretchr/testify/assert" ) func TestMimeAlias(t *testing.T) { @@ -20,3 +21,108 @@ func TestMimeAlias(t *testing.T) { assert.Equal(t, d.out, mimeAlias(d.in)) } } + +func TestMimeType(t *testing.T) { + s := &Source{URL: mustParseURL("http://example.com/list?type=a/b/c")} + _, err := s.mimeType("") + assert.Error(t, err) + + data := []struct { + url string + mediaType string + expected string + }{ + {"http://example.com/foo.json", + "", + jsonMimetype}, + {"http://example.com/foo.json", + "text/foo", + "text/foo"}, + {"http://example.com/foo.json?type=application/yaml", + "text/foo", + "application/yaml"}, + {"http://example.com/list?type=application/array%2Bjson", + "text/foo", + "application/array+json"}, + {"http://example.com/list?type=application/array+json", + "", + "application/array+json"}, + {"http://example.com/unknown", + "", + "text/plain"}, + } + + for i, d := range data { + d := d + t.Run(fmt.Sprintf("%d:%q,%q==%q", i, d.url, d.mediaType, d.expected), func(t *testing.T) { + s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} + mt, err := s.mimeType("") + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + + mt, err = guessMimeType(mustParseURL(d.url), "", d.mediaType) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + }) + } +} + +func TestMimeTypeWithArg(t *testing.T) { + s := &Source{URL: mustParseURL("http://example.com")} + _, err := s.mimeType("h\nttp://foo") + assert.Error(t, err) + + data := []struct { + url string + mediaType string + arg string + expected string + }{ + {"http://example.com/unknown", + "", + "/foo.json", + "application/json"}, + {"http://example.com/unknown", + "", + "foo.json", + "application/json"}, + {"http://example.com/", + "text/foo", + "/foo.json", + "text/foo"}, + {"git+https://example.com/myrepo", + "", + "//foo.yaml", + "application/yaml"}, + {"http://example.com/foo.json", + "", + "/foo.yaml", + "application/yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "", + "/foo.yaml", + "application/array+yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "", + "/foo.yaml?type=application/yaml", + "application/yaml"}, + {"http://example.com/foo.json?type=application/array+yaml", + "text/plain", + "/foo.yaml?type=application/yaml", + "application/yaml"}, + } + + for i, d := range data { + d := d + t.Run(fmt.Sprintf("%d:%q,%q,%q==%q", i, d.url, d.mediaType, d.arg, d.expected), func(t *testing.T) { + s := &Source{URL: mustParseURL(d.url), mediaType: d.mediaType} + mt, err := s.mimeType(d.arg) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + + mt, err = guessMimeType(mustParseURL(d.url), d.arg, d.mediaType) + assert.NoError(t, err) + assert.Equal(t, d.expected, mt) + }) + } +} diff --git a/go.mod b/go.mod index 359a85de0..540ff8bd1 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,6 @@ require ( github.com/aws/aws-sdk-go v1.44.37 github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa - github.com/go-git/go-billy/v5 v5.3.1 - github.com/go-git/go-git/v5 v5.4.2 github.com/google/uuid v1.3.0 github.com/gosimple/slug v1.12.0 github.com/hairyhenderson/go-fsimpl v0.0.0-20220529183339-9deae3e35047 @@ -27,7 +25,6 @@ require ( github.com/stretchr/testify v1.7.2 github.com/ugorji/go/codec v1.2.7 github.com/zealic/xignore v0.3.3 - gocloud.dev v0.25.1-0.20220408200107-09b10f7359f7 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 @@ -46,6 +43,17 @@ require ( cloud.google.com/go/compute v1.6.1 // indirect cloud.google.com/go/iam v0.3.0 // indirect cloud.google.com/go/storage v1.22.1 // indirect + github.com/Azure/azure-pipeline-go v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect + github.com/Azure/azure-storage-blob-go v0.14.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.24 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-crypto v0.0.0-20220517143526-88bb52951d5b // indirect github.com/acomagu/bufpipe v1.0.3 // indirect @@ -66,6 +74,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.5 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.5 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10 // indirect + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.9 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.6 // indirect github.com/aws/smithy-go v1.11.2 // indirect @@ -75,6 +84,9 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.13.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect + github.com/go-git/go-git/v5 v5.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -107,6 +119,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-ieproxy v0.0.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -126,6 +139,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go4.org/intern v0.0.0-20220301175310-a089fc204883 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect + gocloud.dev v0.25.1-0.20220408200107-09b10f7359f7 // indirect golang.org/x/net v0.0.0-20220526153639-5463443f8c37 // indirect golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect diff --git a/go.sum b/go.sum index 70cc67d30..1eb5f4199 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.19.0/go.mod h1:/O9kmSe9bb9KRnIAWkzmqhPjHo6LtzGOBYd/kr06XSs= +cloud.google.com/go/pubsub v1.21.1 h1:ghu6wlm6WouITmmuwkxGG+6vNRXDaPdAjqLcRdsw3EQ= cloud.google.com/go/secretmanager v1.3.0/go.mod h1:+oLTkouyiYiabAQNugCeTS3PAArGiMJuBqvJnJsyH+U= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -85,28 +86,45 @@ github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v59.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/azure-service-bus-go v0.11.5/go.mod h1:MI6ge2CuQWBVq+ly456MY7XqNLJip5LO1iSFodbNLbU= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-amqp v0.16.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= github.com/Azure/go-amqp v0.16.4/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.22/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= +github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.17/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.9 h1:Y2CgdzitFDsdMwYMzf9LIZWrrTFysqbRc7b94XVVJ78= github.com/Azure/go-autorest/autorest/azure/auth v0.5.9/go.mod h1:hg3/1yw0Bq87O3KvvnJoAh34/0zbP7SFizX/qN5JvjU= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -200,6 +218,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3/go.mod h1:g1qvDuRsJY+XghsV6zg00Z github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10 h1:GWdLZK0r1AK5sKb8rhB9bEXqXCK8WNuyv4TBAD6ZviQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.26.10/go.mod h1:+O7qJxF8nLorAhuIVhYTHse6okjHJJm4EwhhzvpnkT0= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU8rrre0/4a0pn2wgwiDvOEzoOjcJUBr67o= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.9 h1:a7+ZYQbKAziY5a7H8Ggwp/6HM9UKT6h9al+QHY+P6jI= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.9/go.mod h1:Jt1lSw1fYlQ60lqrZ9ViN2LMGizbWTWbkStm4rbuYuE= github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw= github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM= github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0= @@ -251,7 +271,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/libkv v0.2.2-0.20180912205406-458977154600 h1:x0AMRhackzbivKKiEeSMzH6gZmbALPXCBG0ecBmRlco= github.com/docker/libkv v0.2.2-0.20180912205406-458977154600/go.mod h1:r5hEwHwW8dr0TFBYGCarMNbrQOiwL1xoqDYZ/JqoTK0= @@ -281,6 +304,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -289,6 +313,7 @@ github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsouza/fake-gcs-server v1.37.12 h1:eMjZgs2KXoN/QmbFuydN29jWwpdioPQcg92dzis21TE= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= @@ -342,6 +367,8 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= @@ -451,6 +478,8 @@ github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -646,6 +675,7 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-ieproxy v0.0.6 h1:tVDlituRyeHMMkHpGpUu8CJG+hxPMwbYCkIUK2PUCbo= +github.com/mattn/go-ieproxy v0.0.6/go.mod h1:6ZpRmhBaYuBX1U2za+9rC9iCGLsSp2tftelZne7CPko= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -712,6 +742,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -758,6 +789,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -857,6 +889,7 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= @@ -932,6 +965,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -945,8 +979,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -990,6 +1026,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1072,6 +1109,7 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220110181412-a018aaa089fe/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/datafs/envfs.go b/internal/datafs/envfs.go new file mode 100644 index 000000000..e0979baa6 --- /dev/null +++ b/internal/datafs/envfs.go @@ -0,0 +1,210 @@ +package datafs + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "strings" + "time" + + "github.com/hairyhenderson/go-fsimpl" +) + +// NewEnvFS returns a filesystem (an fs.FS) that can be used to read data from +// environment variables. +func NewEnvFS(u *url.URL) (fs.FS, error) { + return &envFS{locfs: os.DirFS("/")}, nil +} + +type envFS struct { + locfs fs.FS +} + +//nolint:gochecknoglobals +var EnvFS = fsimpl.FSProviderFunc(NewEnvFS, "env") + +var _ fs.FS = (*envFS)(nil) + +func (f *envFS) Open(name string) (fs.File, error) { + if !fs.ValidPath(name) { + return nil, &fs.PathError{ + Op: "open", + Path: name, + Err: fs.ErrInvalid, + } + } + + return &envFile{locfs: f.locfs, name: name}, nil +} + +type envFile struct { + locfs fs.FS + body io.Reader + name string + + dirents []fs.DirEntry + diroff int +} + +var ( + _ fs.File = (*envFile)(nil) + _ fs.ReadDirFile = (*envFile)(nil) +) + +func (e *envFile) Close() error { + e.body = nil + return nil +} + +func (e *envFile) envReader() (int, io.Reader, error) { + v, found := os.LookupEnv(e.name) + if found { + return len(v), bytes.NewBufferString(v), nil + } + + fname, found := os.LookupEnv(e.name + "_FILE") + if found && fname != "" { + fname = strings.TrimPrefix(fname, "/") + + b, err := fs.ReadFile(e.locfs, fname) + if err != nil { + return 0, nil, err + } + + b = bytes.TrimSpace(b) + + return len(b), bytes.NewBuffer(b), nil + } + + return 0, nil, fs.ErrNotExist +} + +func (e *envFile) Stat() (fs.FileInfo, error) { + n, _, err := e.envReader() + if err != nil { + return nil, err + } + + return FileInfo(e.name, int64(n), 0444, time.Time{}, ""), nil +} + +func (e *envFile) Read(p []byte) (int, error) { + if e.body == nil { + _, r, err := e.envReader() + if err != nil { + return 0, err + } + e.body = r + } + + return e.body.Read(p) +} + +func (e *envFile) ReadDir(n int) ([]fs.DirEntry, error) { + // envFS has no concept of subdirectories, but we can support a root + // directory by listing all environment variables. + if e.name != "." { + return nil, fmt.Errorf("%s: is not a directory", e.name) + } + + if e.dirents == nil { + envs := os.Environ() + e.dirents = make([]fs.DirEntry, len(envs)) + for i, env := range envs { + parts := strings.SplitN(env, "=", 2) + name, value := parts[0], parts[1] + + e.dirents[i] = FileInfoDirEntry( + FileInfo(name, int64(len(value)), 0444, time.Time{}, ""), + ) + } + } + + if n > 0 && e.diroff >= len(e.dirents) { + return nil, io.EOF + } + + low := e.diroff + high := e.diroff + n + + // clamp high at the max, and ensure it's higher than low + if high >= len(e.dirents) || high <= low { + high = len(e.dirents) + } + + entries := make([]fs.DirEntry, high-low) + copy(entries, e.dirents[e.diroff:]) + + e.diroff = high + + return entries, nil +} + +// FileInfo/DirInfo/FileInfoDirEntry/etc are taken from go-fsimpl's internal +// package, and may be exported in the future... + +// FileInfo creates a static fs.FileInfo with the given properties. +// The result is also a fs.DirEntry and can be safely cast. +func FileInfo(name string, size int64, mode fs.FileMode, modTime time.Time, contentType string) fs.FileInfo { + return &staticFileInfo{ + name: name, + size: size, + mode: mode, + modTime: modTime, + contentType: contentType, + } +} + +// DirInfo creates a fs.FileInfo for a directory with the given name. Use +// FileInfo to set other values. +func DirInfo(name string, modTime time.Time) fs.FileInfo { + return FileInfo(name, 0, fs.ModeDir, modTime, "") +} + +type staticFileInfo struct { + modTime time.Time + name string + contentType string + size int64 + mode fs.FileMode +} + +var ( + _ fs.FileInfo = (*staticFileInfo)(nil) + _ fs.DirEntry = (*staticFileInfo)(nil) +) + +func (fi staticFileInfo) ContentType() string { return fi.contentType } +func (fi staticFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi staticFileInfo) Mode() fs.FileMode { return fi.mode } +func (fi *staticFileInfo) ModTime() time.Time { return fi.modTime } +func (fi staticFileInfo) Name() string { return fi.name } +func (fi staticFileInfo) Size() int64 { return fi.size } +func (fi staticFileInfo) Sys() interface{} { return nil } +func (fi *staticFileInfo) Info() (fs.FileInfo, error) { return fi, nil } +func (fi staticFileInfo) Type() fs.FileMode { return fi.Mode().Type() } + +// FileInfoDirEntry adapts a fs.FileInfo into a fs.DirEntry. If it doesn't +// already implement fs.DirEntry, it will be wrapped to always return the +// same fs.FileInfo. +func FileInfoDirEntry(fi fs.FileInfo) fs.DirEntry { + de, ok := fi.(fs.DirEntry) + if ok { + return de + } + + return &fileinfoDirEntry{fi} +} + +// a wrapper to make a fs.FileInfo into an fs.DirEntry +type fileinfoDirEntry struct { + fs.FileInfo +} + +var _ fs.DirEntry = (*fileinfoDirEntry)(nil) + +func (fi *fileinfoDirEntry) Info() (fs.FileInfo, error) { return fi, nil } +func (fi *fileinfoDirEntry) Type() fs.FileMode { return fi.Mode().Type() } diff --git a/internal/datafs/envfs_test.go b/internal/datafs/envfs_test.go new file mode 100644 index 000000000..3f41c129f --- /dev/null +++ b/internal/datafs/envfs_test.go @@ -0,0 +1,105 @@ +package datafs + +import ( + "io/fs" + "net/url" + "os" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" +) + +func TestEnvFS_Open(t *testing.T) { + fsys, err := NewEnvFS(nil) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + f, err := fsys.Open("foo") + assert.NoError(t, err) + assert.IsType(t, &envFile{}, f) +} + +func TestEnvFile_Read(t *testing.T) { + content := `hello world` + os.Setenv("HELLO_WORLD", "hello world") + defer os.Unsetenv("HELLO_WORLD") + + f := &envFile{name: "HELLO_WORLD"} + b := make([]byte, len(content)) + n, err := f.Read(b) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + f = &envFile{name: "FOO", locfs: fsys} + + b = make([]byte, len(content)) + t.Logf("b len is %d", len(b)) + n, err = f.Read(b) + t.Logf("b len is %d", len(b)) + assert.NoError(t, err) + assert.Equal(t, len(content), n) + assert.Equal(t, content, string(b)) +} + +func TestEnvFile_Stat(t *testing.T) { + content := []byte(`hello world`) + os.Setenv("HELLO_WORLD", "hello world") + defer os.Unsetenv("HELLO_WORLD") + + f := &envFile{name: "HELLO_WORLD"} + + fi, err := f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) + + fsys := fstest.MapFS{} + fsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello world\n")} + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + f = &envFile{name: "FOO", locfs: fsys} + + fi, err = f.Stat() + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), fi.Size()) +} + +func TestEnvFS(t *testing.T) { + u, _ := url.Parse("env:") + + lfsys := fstest.MapFS{} + lfsys["foo/bar/baz.txt"] = &fstest.MapFile{Data: []byte("\nhello file\n")} + + fsys, err := NewEnvFS(u) + assert.NoError(t, err) + assert.IsType(t, &envFS{}, fsys) + + envfs, ok := fsys.(*envFS) + assert.True(t, ok) + envfs.locfs = lfsys + + os.Setenv("FOO_FILE", "/foo/bar/baz.txt") + defer os.Unsetenv("FOO_FILE") + + b, err := fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello file", string(b)) + + os.Setenv("FOO", "hello world") + defer os.Unsetenv("FOO") + + b, err = fs.ReadFile(fsys, "FOO") + assert.NoError(t, err) + assert.Equal(t, "hello world", string(b)) + + assert.NoError(t, fstest.TestFS(fsys, "FOO", "FOO_FILE", "HOME", "USER")) +} diff --git a/internal/tests/integration/datasources_vault_test.go b/internal/tests/integration/datasources_vault_test.go index 68ecafacb..e138f9436 100644 --- a/internal/tests/integration/datasources_vault_test.go +++ b/internal/tests/integration/datasources_vault_test.go @@ -4,6 +4,7 @@ package integration import ( + "fmt" "os" "os/user" "path" @@ -63,7 +64,7 @@ func startVault(t *testing.T) (*fs.Dir, *vaultClient) { "-dev", "-dev-root-token-id="+vaultRootToken, "-dev-leased-kv", - "-log-level=err", + "-log-level=trace", "-dev-listen-address="+vaultAddr, "-config="+tmpDir.Join("config.json"), ) @@ -85,6 +86,8 @@ func startVault(t *testing.T) (*fs.Dir, *vaultClient) { result.Assert(t, icmd.Expected{ExitCode: 0}) + fmt.Println(result.Combined()) + // restore old token if it was backed up u, _ := user.Current() homeDir := u.HomeDir