From fe12fd051e4d1b808c730d5b17c3c33ee885a927 Mon Sep 17 00:00:00 2001 From: Johnny Steenbergen Date: Sat, 11 Jan 2020 18:49:55 -0800 Subject: [PATCH] feat(pkger): add jsonnet support for package files --- CHANGELOG.md | 1 + cmd/influx/pkg.go | 14 ++---- go.mod | 1 + go.sum | 6 +++ http/pkger_http_server.go | 4 ++ http/pkger_http_server_test.go | 32 +++++++++++++ http/swagger.yml | 3 ++ pkg/jsonnet/decode.go | 34 +++++++++++++ pkg/jsonnet/decode_test.go | 48 +++++++++++++++++++ pkger/parser.go | 18 ++++++- pkger/parser_test.go | 46 ++++++++++++++++++ .../testdata/bucket_associates_labels.jsonnet | 39 +++++++++++++++ 12 files changed, 235 insertions(+), 11 deletions(-) create mode 100644 pkg/jsonnet/decode.go create mode 100644 pkg/jsonnet/decode_test.go create mode 100644 pkger/testdata/bucket_associates_labels.jsonnet diff --git a/CHANGELOG.md b/CHANGELOG.md index c19a8652472..9b0fdbe7015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ 1. [16260](https://github.com/influxdata/influxdb/pull/16260): Capture User-Agent header as query source for logging purposes 1. [16469](https://github.com/influxdata/influxdb/pull/16469): Add support for configurable max batch size in points write handler 1. [16509](https://github.com/influxdata/influxdb/pull/16509): Add support for applying an influx package via a public facing URL +1. [16511](https://github.com/influxdata/influxdb/pull/16511): Add jsonnet support for influx packages ### Bug Fixes diff --git a/cmd/influx/pkg.go b/cmd/influx/pkg.go index a73f2f15732..66bd76dcb98 100644 --- a/cmd/influx/pkg.go +++ b/cmd/influx/pkg.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" "path/filepath" "reflect" @@ -95,7 +94,7 @@ func (b *cmdPkgBuilder) cmdPkgApply() *cobra.Command { cmd.Short = "Apply a pkg to create resources" cmd.Flags().StringVarP(&b.file, "file", "f", "", "Path to package file") - cmd.MarkFlagFilename("file", "yaml", "yml", "json") + cmd.MarkFlagFilename("file", "yaml", "yml", "json", "jsonnet") cmd.Flags().BoolVarP(&b.quiet, "quiet", "q", false, "disable output printing") cmd.Flags().StringVar(&b.applyOpts.force, "force", "", `TTY input, if package will have destructive changes, proceed if set "true"`) cmd.Flags().StringVarP(&b.applyOpts.url, "url", "u", "", "URL to retrieve a package.") @@ -551,14 +550,7 @@ func pkgFromReader(stdin io.Reader) (*pkger.Pkg, error) { return nil, err } - var enc pkger.Encoding - switch http.DetectContentType(b[0:512]) { - case "application/json": - enc = pkger.EncodingJSON - default: - enc = pkger.EncodingYAML - } - return pkger.Parse(enc, pkger.FromString(string(b))) + return pkger.Parse(pkger.EncodingSource, pkger.FromString(string(b))) } func pkgFromFile(path string) (*pkger.Pkg, error) { @@ -568,6 +560,8 @@ func pkgFromFile(path string) (*pkger.Pkg, error) { enc = pkger.EncodingYAML case ".json": enc = pkger.EncodingJSON + case ".jsonnet": + enc = pkger.EncodingJsonnet default: return nil, errors.New("file provided must be one of yaml/yml/json extension but got: " + ext) } diff --git a/go.mod b/go.mod index 6e7fc6b188f..de58e998302 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/google/btree v1.0.0 github.com/google/go-cmp v0.3.1 github.com/google/go-github v17.0.0+incompatible + github.com/google/go-jsonnet v0.14.0 github.com/goreleaser/goreleaser v0.97.0 github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c // indirect github.com/hashicorp/raft v1.0.0 // indirect diff --git a/go.sum b/go.sum index 4a56882417c..0b5547af0a1 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,8 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-jsonnet v0.14.0 h1:as/sAfmjOHqY/OMBR4mv9I8ZY0/jNuqN3u44AicwxPs= +github.com/google/go-jsonnet v0.14.0/go.mod h1:zPGC9lj/TbjkBtUACIvYR/ILHrFqKRhxeEA+bLyeMnY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -288,11 +290,14 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= @@ -539,6 +544,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/http/pkger_http_server.go b/http/pkger_http_server.go index ba76c7cbc0a..b9df6413afc 100644 --- a/http/pkger_http_server.go +++ b/http/pkger_http_server.go @@ -12,6 +12,7 @@ import ( "github.com/influxdata/influxdb" pctx "github.com/influxdata/influxdb/context" "github.com/influxdata/influxdb/pkg/httpc" + "github.com/influxdata/influxdb/pkg/jsonnet" "github.com/influxdata/influxdb/pkger" "go.uber.org/zap" "gopkg.in/yaml.v3" @@ -232,6 +233,9 @@ func decodeWithEncoding(r *http.Request, v interface{}) (pkger.Encoding, error) dec interface{ Decode(interface{}) error } ) switch contentType := r.Header.Get("Content-Type"); contentType { + case "application/x-jsonnet": + encoding = pkger.EncodingJsonnet + dec = jsonnet.NewDecoder(r.Body) case "text/yml", "application/x-yaml": encoding = pkger.EncodingYAML dec = yaml.NewDecoder(r.Body) diff --git a/http/pkger_http_server_test.go b/http/pkger_http_server_test.go index d3deddd7227..f6758abe836 100644 --- a/http/pkger_http_server_test.go +++ b/http/pkger_http_server_test.go @@ -103,6 +103,15 @@ func TestPkgerHTTPServer(t *testing.T) { URL: "https://gist.githubusercontent.com/jsteenb2/3a3b2b5fcbd6179b2494c2b54aa2feb0/raw/1717709ffadbeed5dfc88ff4cac5bf912c6930bf/bucket_pkg_json", }, }, + { + name: "app jsonnet", + contentType: "application/x-jsonnet", + reqBody: fluxTTP.ReqApplyPkg{ + DryRun: true, + OrgID: influxdb.ID(9000).String(), + Pkg: bucketPkg(t, pkger.EncodingJsonnet), + }, + }, } for _, tt := range tests { @@ -257,6 +266,29 @@ func bucketPkg(t *testing.T, encoding pkger.Encoding) *pkger.Pkg { var pkgStr string switch encoding { + case pkger.EncodingJsonnet: + pkgStr = ` +local Bucket(name, desc) = { + kind: 'Bucket', + name: name, + description: desc, +}; + +{ + apiVersion: "0.1.0", + kind: "Package", + meta: { + pkgName: "pkg_name", + pkgVersion: "1", + description: "pack description" + }, + spec: { + resources: [ + Bucket(name="rucket_1", desc="bucket 1 description"), + ] + } +} +` case pkger.EncodingJSON: pkgStr = ` { diff --git a/http/swagger.yml b/http/swagger.yml index 2233c34cff7..b461bca2baa 100644 --- a/http/swagger.yml +++ b/http/swagger.yml @@ -4319,6 +4319,9 @@ paths: application/json: schema: $ref: "#/components/schemas/PkgApply" + application/x-jsonnet: + schema: + $ref: "#/components/schemas/PkgApply" text/yml: schema: $ref: "#/components/schemas/PkgApply" diff --git a/pkg/jsonnet/decode.go b/pkg/jsonnet/decode.go new file mode 100644 index 00000000000..c175cbceafd --- /dev/null +++ b/pkg/jsonnet/decode.go @@ -0,0 +1,34 @@ +package jsonnet + +import ( + "encoding/json" + "io" + "io/ioutil" + + "github.com/google/go-jsonnet" +) + +// Decoder type can decoce a jsonnet stream into the given output. +type Decoder struct { + r io.Reader +} + +// NewDecoder creates a new decoder. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +// Decode decodes the stream into the provide value. +func (d *Decoder) Decode(v interface{}) error { + b, err := ioutil.ReadAll(d.r) + if err != nil { + return err + } + + vm := jsonnet.MakeVM() + jsonStr, err := vm.EvaluateSnippet("memory", string(b)) + if err != nil { + return err + } + return json.Unmarshal([]byte(jsonStr), &v) +} diff --git a/pkg/jsonnet/decode_test.go b/pkg/jsonnet/decode_test.go new file mode 100644 index 00000000000..51e604bd67f --- /dev/null +++ b/pkg/jsonnet/decode_test.go @@ -0,0 +1,48 @@ +package jsonnet_test + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/influxdata/influxdb/pkg/jsonnet" + "github.com/stretchr/testify/require" +) + +func TestDecoder(t *testing.T) { + type ( + person struct { + Name string `json:"name"` + Welcome string `json:"welcome"` + } + + persons struct { + Person1 person `json:"person1"` + Person2 person `json:"person2"` + } + ) + + const entry = `{ + person1: { + name: "Alice", + welcome: "Hello " + self.name + "!", + }, + person2: self.person1 { name: "Bob" }, +}` + + var out persons + require.NoError(t, jsonnet.NewDecoder(strings.NewReader(entry)).Decode(&out)) + + expected := persons{ + Person1: person{ + Name: "Alice", + Welcome: "Hello Alice!", + }, + Person2: person{ + Name: "Bob", + Welcome: "Hello Bob!", + }, + } + assert.Equal(t, expected, out) +} diff --git a/pkger/parser.go b/pkger/parser.go index e98777001b4..d37a89cd120 100644 --- a/pkger/parser.go +++ b/pkger/parser.go @@ -14,6 +14,7 @@ import ( "time" "github.com/influxdata/influxdb" + "github.com/influxdata/influxdb/pkg/jsonnet" "gopkg.in/yaml.v3" ) @@ -29,6 +30,7 @@ type Encoding int const ( EncodingUnknown Encoding = iota EncodingJSON + EncodingJsonnet EncodingSource // EncodingSource draws the encoding type by inferring it from the source. EncodingYAML ) @@ -38,6 +40,8 @@ func (e Encoding) String() string { switch e { case EncodingJSON: return "json" + case EncodingJsonnet: + return "jsonnet" case EncodingSource: return "source" case EncodingYAML: @@ -61,6 +65,8 @@ func Parse(encoding Encoding, readerFn ReaderFn, opts ...ValidateOptFn) (*Pkg, e switch encoding { case EncodingJSON: return parseJSON(r, opts...) + case EncodingJsonnet: + return parseJsonnet(r, opts...) case EncodingSource: return parseSource(r, opts...) case EncodingYAML: @@ -122,6 +128,10 @@ func parseJSON(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { return parse(json.NewDecoder(r), opts...) } +func parseJsonnet(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { + return parse(jsonnet.NewDecoder(r), opts...) +} + func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { var b []byte if byter, ok := r.(interface{ Bytes() []byte }); ok { @@ -136,10 +146,16 @@ func parseSource(r io.Reader, opts ...ValidateOptFn) (*Pkg, error) { contentType := http.DetectContentType(b) switch { + case strings.Contains(contentType, "jsonnet"): + // highly unlikely to fall in here with supported content type detection as is + return parseJsonnet(bytes.NewReader(b), opts...) case strings.Contains(contentType, "json"): return parseJSON(bytes.NewReader(b), opts...) - default: + case strings.Contains(contentType, "yaml"), + strings.Contains(contentType, "yml"): return parseYAML(bytes.NewReader(b), opts...) + default: + return parseJsonnet(r, opts...) } } diff --git a/pkger/parser_test.go b/pkger/parser_test.go index bbc5731c98e..2246157106f 100644 --- a/pkger/parser_test.go +++ b/pkger/parser_test.go @@ -4338,6 +4338,52 @@ spec: require.True(t, ok) }) }) + + t.Run("jsonnet support", func(t *testing.T) { + pkg := validParsedPkg(t, "testdata/bucket_associates_labels.jsonnet", EncodingJsonnet, baseAsserts{ + version: "0.1.0", + kind: KindPackage, + description: "pack description", + metaName: "pkg_name", + metaVersion: "1", + }) + + sum := pkg.Summary() + + labels := []SummaryLabel{ + { + Name: "label_1", + Properties: struct { + Color string `json:"color"` + Description string `json:"description"` + }{Color: "#eee888", Description: "desc_1"}, + }, + } + assert.Equal(t, labels, sum.Labels) + + bkts := []SummaryBucket{ + { + Name: "rucket_1", + Description: "desc_1", + RetentionPeriod: 10000 * time.Second, + LabelAssociations: labels, + }, + { + Name: "rucket_2", + Description: "desc_2", + RetentionPeriod: 20000 * time.Second, + LabelAssociations: labels, + }, + { + Name: "rucket_3", + Description: "desc_3", + RetentionPeriod: 30000 * time.Second, + LabelAssociations: labels, + }, + } + assert.Equal(t, bkts, sum.Buckets) + }) + } func Test_IsParseError(t *testing.T) { diff --git a/pkger/testdata/bucket_associates_labels.jsonnet b/pkger/testdata/bucket_associates_labels.jsonnet new file mode 100644 index 00000000000..13d82c85813 --- /dev/null +++ b/pkger/testdata/bucket_associates_labels.jsonnet @@ -0,0 +1,39 @@ +local Label(name, desc, color) = { + kind: 'Label', + name: name, + description: desc, + color: color +}; + +local LabelAssociations(names=[]) = [ + {kind: 'Label', name: name} + for name in names +]; + +local Bucket(name, desc, secs, associations=LabelAssociations(['label_1'])) = { + kind: 'Bucket', + name: name, + description: desc, + retentionRules: [ + {type: 'expire', everySeconds: secs} + ], + associations: associations +}; + +{ + apiVersion: "0.1.0", + kind: "Package", + meta: { + pkgName: "pkg_name", + pkgVersion: "1", + description: "pack description" + }, + spec: { + resources: [ + Label("label_1",desc="desc_1", color='#eee888'), + Bucket(name="rucket_1", desc="desc_1", secs=10000), + Bucket("rucket_2", "desc_2", 20000), + Bucket("rucket_3", "desc_3", 30000), + ] + } +}