From 4898a07239b86636c3cb3bfe927831aeec0ae14b Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Tue, 1 Aug 2023 19:36:01 +0200 Subject: [PATCH] enhancement: add config option to load an RFC 4288 mimes file --- ...dable-policy-mimetype-extension-mapping.md | 8 ++ go.mod | 3 +- go.sum | 2 - services/policies/README.md | 36 +++++-- services/policies/pkg/config/config.go | 2 + services/policies/pkg/engine/opa/engine.go | 37 ++++--- .../policies/pkg/engine/opa/rf_mimetype.go | 97 ++++++++++++------- .../pkg/engine/opa/rf_mimetype_test.go | 40 ++++++-- .../policies/pkg/engine/opa/rf_resource.go | 4 + vendor/modules.txt | 3 - 10 files changed, 166 insertions(+), 66 deletions(-) create mode 100644 changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md diff --git a/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md b/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md new file mode 100644 index 00000000000..d33c4adeb5c --- /dev/null +++ b/changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md @@ -0,0 +1,8 @@ +Enhancement: Extendable policy mimetype extension mapping + +The extension mimetype mappings known from rego can now be extended. +To do this, ocis must be informed where the mimetype file (apache mime.types file format) is located. + +`export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types` + +https://github.com/owncloud/ocis/pull/6869 diff --git a/go.mod b/go.mod index 9a7e002c0e6..90a3d83b82e 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 + github.com/gabriel-vasile/mimetype v1.4.2 github.com/ggwhite/go-masker v1.0.9 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 @@ -67,7 +68,6 @@ require ( github.com/pkg/xattr v0.4.9 github.com/prometheus/client_golang v1.16.0 github.com/r3labs/sse/v2 v2.10.0 - github.com/rakyll/magicmime v0.1.0 github.com/riandyrn/otelchi v0.5.1 github.com/rogpeppe/go-internal v1.10.0 github.com/rs/zerolog v1.29.1 @@ -176,7 +176,6 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gdexlab/go-render v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-acme/lego/v4 v4.4.0 // indirect diff --git a/go.sum b/go.sum index 2e55700c2ce..2a921a5a0cc 100644 --- a/go.sum +++ b/go.sum @@ -1477,8 +1477,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= -github.com/rakyll/magicmime v0.1.0 h1:aFIp1DqgzjcB3FI7rQk6uZl73i1VPpWswab1YKU4CL4= -github.com/rakyll/magicmime v0.1.0/go.mod h1:OKs4S+1GpIAB1PCebhwp3rxhyipe7TiImiIeVyFlQt8= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/services/policies/README.md b/services/policies/README.md index 4dccbf52d85..5e43041f7f0 100644 --- a/services/policies/README.md +++ b/services/policies/README.md @@ -20,7 +20,7 @@ To configure the policies service, three environment variables need to be define Note that each query setting defines the [Complete Rules](https://www.openpolicyagent.org/docs/latest/#complete-rules) variable defined in the rego rule set the corresponding step uses for the evaluation. If the variable is mistyped or not found, the evaluation defaults to deny. Individual query definitions can be defined for each module. -To activate a the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny. +To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny. When using async post-processing which is done via the postprocessing service, the value `policies` must be added to the `POSTPROCESSING_STEPS` configuration in postprocessing service in the order where the evaluation should take place. @@ -88,8 +88,8 @@ proxy: The same can be achieved by setting the following environment variable: -```yaml -PROXY_POLICIES_QUERY=data.proxy.granted +```shell +export PROXY_POLICIES_QUERY=data.proxy.granted ``` ### Postprocessing @@ -102,14 +102,14 @@ policies: The same can be achieved by setting the following environment variable: -```yaml -POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted +```shell +export POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted ``` As soon as that query is configured, the postprocessing service must be informed to use the policies step by setting the environment variable: -```yaml -POSTPROCESSING_STEPS=policies +```shell +export POSTPROCESSING_STEPS=policies ``` Note that additional steps can be configured and their position in the list defines the order of processing. For details see the postprocessing service documentation. @@ -118,6 +118,28 @@ Note that additional steps can be configured and their position in the list defi To identify available keys for OPA, you need to look at [engine.go](https://github.com/owncloud/ocis/blob/master/services/policies/pkg/engine/engine.go) and the [policies.swagger.json](https://github.com/owncloud/ocis/blob/master/protogen/gen/ocis/services/policies/v0/policies.swagger.json) file. Note that which keys are available depends on from which module it is used. +### Extend mimetype file extension mapping + +In rego it is possible to get a list of associated file extensions based on a mimetype, e.g. 'ocis.mimetype.extensions("application/pdf")'. + +The list of mappings is restricted by default and is provided by the host system. + +In order to extend this list, oCis must be provided with the path to a mime.types file. + +This can be done via yaml configuration or an environment variable. + +```shell +export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types +``` + +```yaml +policies: + engine: + mimes: OCIS_HOME/mime.types +``` + +A good example of how such a file should be formatted can be found in the [apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types). + ## Example Policies The policies service contains a set of preconfigured example policies. See the [deployment examples](https://github.com/owncloud/ocis/tree/master/deployments/examples) directory for details. The contained policies disallow Infinite Scale to create certain file types, both via the proxy middleware and the events service via postprocessing. diff --git a/services/policies/pkg/config/config.go b/services/policies/pkg/config/config.go index 3eb9b05f0d1..9fb5df70ff5 100644 --- a/services/policies/pkg/config/config.go +++ b/services/policies/pkg/config/config.go @@ -45,6 +45,8 @@ type TokenManager struct { type Engine struct { Timeout time.Duration `yaml:"timeout" env:"POLICIES_ENGINE_TIMEOUT" desc:"Sets the timeout the rego expression evaluation can take. The timeout can be set as number followed by a unit identifier like ms, s, etc. Rules default to deny if the timeout was reached."` Policies []string `yaml:"policies"` + // Mimes file path, RFC 4288 + Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps media types to unique file extension(s)."` } // Postprocessing defines the config options for the postprocessing policy handling. diff --git a/services/policies/pkg/engine/opa/engine.go b/services/policies/pkg/engine/opa/engine.go index 45bd211bf79..f152a6403ba 100644 --- a/services/policies/pkg/engine/opa/engine.go +++ b/services/policies/pkg/engine/opa/engine.go @@ -2,6 +2,8 @@ package opa import ( "context" + "io" + "os" "time" "github.com/open-policy-agent/opa/rego" @@ -17,16 +19,35 @@ type OPA struct { printHook print.Hook policies []string timeout time.Duration + options []func(r *rego.Rego) } // NewOPA returns a ready to use opa engine. func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) { + var mtReader io.Reader = nil + if conf.Mimes != "" { + mtReader, err := os.Open(conf.Mimes) + if err != nil { + return OPA{}, err + } + defer mtReader.Close() + } + + rfMimetypeExtensions, err := RFMimetypeExtensions(mtReader) + if err != nil { + return OPA{}, err + } + return OPA{ - policies: conf.Policies, - timeout: timeout, - printHook: logPrinter{logger: logger}, + policies: conf.Policies, + timeout: timeout, + printHook: logPrinter{logger: logger}, + options: []func(r *rego.Rego){ + RFMimetypeDetect, + RFResourceDownload, + rfMimetypeExtensions, }, - nil + }, nil } // Evaluate evaluates the opa policies and returns the result. @@ -34,19 +55,13 @@ func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (b ctx, cancel := context.WithTimeout(ctx, o.timeout) defer cancel() - customFns := []func(r *rego.Rego){ - RFResourceDownload, - RFMimetypeDetect, - RFMimetypeExtensions, - } - q, err := rego.New( append([]func(r *rego.Rego){ rego.Query(qs), rego.Load(o.policies, nil), rego.EnablePrintStatements(true), rego.PrintHook(o.printHook), - }, customFns...)..., + }, o.options...)..., ).PrepareForEval(ctx) if err != nil { return false, err diff --git a/services/policies/pkg/engine/opa/rf_mimetype.go b/services/policies/pkg/engine/opa/rf_mimetype.go index 8e317bb2c71..4fb5b02adea 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype.go +++ b/services/policies/pkg/engine/opa/rf_mimetype.go @@ -1,46 +1,81 @@ package opa import ( - "log" + "bufio" + "io" "mime" "strings" + "github.com/gabriel-vasile/mimetype" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/types" - "github.com/rakyll/magicmime" ) -var RFMimetypeExtensions = rego.Function1( - ®o.Function{ - Name: "ocis.mimetype.extensions", - Decl: types.NewFunction(types.Args(types.S), types.A), - Memoize: true, - Nondeterministic: true, - }, - func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { - var mt string +// RFMimetypeExtensions extends the rego dictionary with the possibility of mapping mimetypes to file extensions. +// Be careful calling this multiple times with individual readers, the mime store is global, +// which results in one global store which holds all known mimetype mappings at once. +// +// Rego: `ocis.mimetype.extensions("application/pdf")` +// Result `[.pdf]` +func RFMimetypeExtensions(f io.Reader) (func(*rego.Rego), error) { + if f != nil { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) <= 1 || fields[0][0] == '#' { + continue + } + mimeType := fields[0] + for _, ext := range fields[1:] { + if ext[0] == '#' { + break + } + if err := mime.AddExtensionType("."+ext, mimeType); err != nil { + return nil, err + } - if err := ast.As(a.Value, &mt); err != nil { - return nil, err + } } - if err := mime.AddExtensionType(".oform", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); err != nil { - return nil, err - } - detectedExtensions, err := mime.ExtensionsByType(mt) - if err != nil { - return nil, err + if err := scanner.Err(); err != nil { + panic(err) } + } - var mimeTerms []*ast.Term - for _, extension := range detectedExtensions { - mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension))) - } + return rego.Function1( + ®o.Function{ + Name: "ocis.mimetype.extensions", + Decl: types.NewFunction(types.Args(types.S), types.A), + Memoize: true, + Nondeterministic: true, + }, + func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { + var mt string - return ast.ArrayTerm(mimeTerms...), nil - }, -) + if err := ast.As(a.Value, &mt); err != nil { + return nil, err + } + + detectedExtensions, err := mime.ExtensionsByType(mt) + if err != nil { + return nil, err + } + + var mimeTerms []*ast.Term + for _, extension := range detectedExtensions { + mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension))) + } + + return ast.ArrayTerm(mimeTerms...), nil + }, + ), nil +} +// RFMimetypeDetect extends the rego dictionary with the possibility to detect mimetypes. +// Be careful, the list of known mimetypes is limited. +// +// Rego: `ocis.mimetype.extensions(".txt")` +// Result `text/plain` var RFMimetypeDetect = rego.Function1( ®o.Function{ Name: "ocis.mimetype.detect", @@ -55,14 +90,8 @@ var RFMimetypeDetect = rego.Function1( return nil, err } - if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil { - log.Fatal(err) - } - defer magicmime.Close() - mimetype, err := magicmime.TypeByBuffer(body) - if err != nil { - return nil, err - } + mimetype := mimetype.Detect(body).String() + return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil }, ) diff --git a/services/policies/pkg/engine/opa/rf_mimetype_test.go b/services/policies/pkg/engine/opa/rf_mimetype_test.go index 404dc2b456d..d2f692deba9 100644 --- a/services/policies/pkg/engine/opa/rf_mimetype_test.go +++ b/services/policies/pkg/engine/opa/rf_mimetype_test.go @@ -2,6 +2,8 @@ package opa_test import ( "context" + "io" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -19,13 +21,37 @@ var _ = Describe("opa ocis mimetype functions", func() { Expect(rs[0].Expressions[0].String()).To(Equal("text/plain")) }) }) + Describe("ocis.mimetype.extensions", func() { + DescribeTable("resolves extensions by mimetype", + func(mimetype string, expectations []string, f io.Reader) { + rfMimetypeExtensions, err := opa.RFMimetypeExtensions(f) + Expect(err).ToNot(HaveOccurred()) - Describe("ocis.mimetype.extension_for_mimetype", func() { - It("provides matching extensions", func() { - r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions) - rs, err := r.Eval(context.Background()) - Expect(err).ToNot(HaveOccurred()) - Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]")) - }) + r := rego.New(rego.Query(`ocis.mimetype.extensions("`+mimetype+`")`), rfMimetypeExtensions) + rs, err := r.Eval(context.Background()) + Expect(err).ToNot(HaveOccurred()) + + got := rs[0].Expressions[0].String() + + if len(expectations) == 0 { + Expect(got).To(Equal("[]")) + } + + for i, expectation := range expectations { + if i+1 != len(expectations) { + expectation += " " + } + + Expect(string(got[0])).To(Equal("[")) + Expect(strings.Contains(got, expectation)).To(BeTrue()) + Expect(string(got[len(got)-1])).To(Equal("]")) + } + }, + Entry("With default mimetype", "application/pdf", []string{".pdf"}, nil), + Entry("With unknown mimetype", "ocis/with.custom.mt", []string{}, nil), + Entry("With custom mimetype", "ocis/with.custom.mt", []string{".with.custom.mt"}, strings.NewReader("ocis/with.custom.mt with.custom.mt")), + Entry("With multiple custom mimetypes", "ocis/with.multiple.custom.mt", []string{".with.multiple.custom.1.mt", ".with.multiple.custom.2.mt"}, strings.NewReader("ocis/with.multiple.custom.mt with.multiple.custom.1.mt with.multiple.custom.2.mt")), + Entry("With custom ignored mimetype", "ocis/with.multiple.custom.ignored.mt", []string{}, strings.NewReader("#ocis/with.multiple.custom.ignored.mt with.multiple.custom.ignored.mt")), + ) }) }) diff --git a/services/policies/pkg/engine/opa/rf_resource.go b/services/policies/pkg/engine/opa/rf_resource.go index 60543c0b83c..9ff1d0ba1ee 100644 --- a/services/policies/pkg/engine/opa/rf_resource.go +++ b/services/policies/pkg/engine/opa/rf_resource.go @@ -11,6 +11,10 @@ import ( "github.com/open-policy-agent/opa/types" ) +// RFResourceDownload extends the rego dictionary with the possibility to download oCis resources. +// +// Rego: `ocis.resource.download("ocis/path/0034892347349827")` +// Result: bytes var RFResourceDownload = rego.Function1( ®o.Function{ Name: "ocis.resource.download", diff --git a/vendor/modules.txt b/vendor/modules.txt index a3bf4e64842..d2d0f2675c4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1565,9 +1565,6 @@ github.com/prometheus/statsd_exporter/pkg/mapper/fsm # github.com/r3labs/sse/v2 v2.10.0 ## explicit; go 1.13 github.com/r3labs/sse/v2 -# github.com/rakyll/magicmime v0.1.0 -## explicit -github.com/rakyll/magicmime # github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 ## explicit github.com/rcrowley/go-metrics