enhancement: add config option to load an RFC 4288 mimes file

changelog/unreleased/enhancement-extendable-policy-mimetype-extension-mapping.md

@@ -0,0 +1,8 @@
+Enhancement: Extendable policy mimetype extension mapping
+
+The extension mimetype mappings known from rego can now be extended.
+To do this, ocis must be informed where the mimetype file (apache mime.types file format) is located.
+
+`export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types`
+
+https://github.com/owncloud/ocis/pull/6869

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
 	github.com/egirna/icap-client v0.1.1
+	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/ggwhite/go-masker v1.0.9
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/go-chi/cors v1.2.1
@@ -67,7 +68,6 @@ require (
 	github.com/pkg/xattr v0.4.9
 	github.com/prometheus/client_golang v1.16.0
 	github.com/r3labs/sse/v2 v2.10.0
-	github.com/rakyll/magicmime v0.1.0
 	github.com/riandyrn/otelchi v0.5.1
 	github.com/rogpeppe/go-internal v1.10.0
 	github.com/rs/zerolog v1.29.1
@@ -176,7 +176,6 @@ require (
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gdexlab/go-render v1.0.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-acme/lego/v4 v4.4.0 // indirect

go.sum

@@ -1477,8 +1477,6 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
 github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0=
 github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I=
 github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA=
-github.com/rakyll/magicmime v0.1.0 h1:aFIp1DqgzjcB3FI7rQk6uZl73i1VPpWswab1YKU4CL4=
-github.com/rakyll/magicmime v0.1.0/go.mod h1:OKs4S+1GpIAB1PCebhwp3rxhyipe7TiImiIeVyFlQt8=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ=
 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=

services/policies/README.md

@@ -20,7 +20,7 @@ To configure the policies service, three environment variables need to be define
 
 Note that each query setting defines the [Complete Rules](https://www.openpolicyagent.org/docs/latest/#complete-rules) variable defined in the rego rule set the corresponding step uses for the evaluation. If the variable is mistyped or not found, the evaluation defaults to deny. Individual query definitions can be defined for each module.
 
-To activate a the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny.
+To activate the policies service for a module, it must be started with a yaml configuration that points to one or more rego files. Note that if the service is scaled horizontally, each instance should have access to the same rego files to avoid unpredictable results. If a file path has been configured but the file it is not present or accessible, the evaluation defaults to deny.
 
 When using async post-processing which is done via the postprocessing service, the value `policies` must be added to the `POSTPROCESSING_STEPS` configuration in postprocessing service in the order where the evaluation should take place.
 
@@ -88,8 +88,8 @@ proxy:
 
 The same can be achieved by setting the following environment variable:
 
-```yaml
-PROXY_POLICIES_QUERY=data.proxy.granted
+```shell
+export PROXY_POLICIES_QUERY=data.proxy.granted
 ```
 
 ### Postprocessing
@@ -102,14 +102,14 @@ policies:
 
 The same can be achieved by setting the following environment variable:
 
-```yaml
-POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted
+```shell
+export POLICIES_POSTPROCESSING_QUERY=data.postprocessing.granted
 ```
 
 As soon as that query is configured, the postprocessing service must be informed to use the policies step by setting the environment variable:
 
-```yaml
-POSTPROCESSING_STEPS=policies
+```shell
+export POSTPROCESSING_STEPS=policies
 ```
 
 Note that additional steps can be configured and their position in the list defines the order of processing. For details see the postprocessing service documentation.
@@ -118,6 +118,28 @@ Note that additional steps can be configured and their position in the list defi
 
 To identify available keys for OPA, you need to look at [engine.go](https://github.com/owncloud/ocis/blob/master/services/policies/pkg/engine/engine.go) and the [policies.swagger.json](https://github.com/owncloud/ocis/blob/master/protogen/gen/ocis/services/policies/v0/policies.swagger.json) file. Note that which keys are available depends on from which module it is used.
 
+### Extend mimetype file extension mapping
+
+In rego it is possible to get a list of associated file extensions based on a mimetype, e.g. 'ocis.mimetype.extensions("application/pdf")'.
+
+The list of mappings is restricted by default and is provided by the host system.
+
+In order to extend this list, oCis must be provided with the path to a mime.types file.
+
+This can be done via yaml configuration or an environment variable.
+
+```shell
+export OCIS_MACHINE_AUTH_API_KEY=$OCIS_HOME/mime.types
+```
+
+```yaml
+policies:
+  engine:
+    mimes: OCIS_HOME/mime.types
+```
+
+A good example of how such a file should be formatted can be found in the [apache svn repository](https://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types).
+
 ## Example Policies
 
 The policies service contains a set of preconfigured example policies. See the [deployment examples](https://github.com/owncloud/ocis/tree/master/deployments/examples) directory for details. The contained policies disallow Infinite Scale to create certain file types, both via the proxy middleware and the events service via postprocessing.

services/policies/pkg/config/config.go

@@ -45,6 +45,8 @@ type TokenManager struct {
 type Engine struct {
 	Timeout  time.Duration `yaml:"timeout" env:"POLICIES_ENGINE_TIMEOUT" desc:"Sets the timeout the rego expression evaluation can take. The timeout can be set as number followed by a unit identifier like ms, s, etc. Rules default to deny if the timeout was reached."`
 	Policies []string      `yaml:"policies"`
+	// Mimes file path, RFC 4288
+	Mimes string `yaml:"mimes" env:"POLICIES_ENGINE_MIMES" desc:"Sets the mimes file path which maps media types to unique file extension(s)."`
 }
 
 // Postprocessing defines the config options for the postprocessing policy handling.

services/policies/pkg/engine/opa/engine.go

@@ -2,6 +2,8 @@ package opa
 
 import (
 	"context"
+	"io"
+	"os"
 	"time"
 
 	"github.com/open-policy-agent/opa/rego"
@@ -17,36 +19,49 @@ type OPA struct {
 	printHook print.Hook
 	policies  []string
 	timeout   time.Duration
+	options   []func(r *rego.Rego)
 }
 
 // NewOPA returns a ready to use opa engine.
 func NewOPA(timeout time.Duration, logger log.Logger, conf config.Engine) (OPA, error) {
+	var mtReader io.Reader = nil
+	if conf.Mimes != "" {
+		mtReader, err := os.Open(conf.Mimes)
+		if err != nil {
+			return OPA{}, err
+		}
+		defer mtReader.Close()
+	}
+
+	rfMimetypeExtensions, err := RFMimetypeExtensions(mtReader)
+	if err != nil {
+		return OPA{}, err
+	}
+
 	return OPA{
-			policies:  conf.Policies,
-			timeout:   timeout,
-			printHook: logPrinter{logger: logger},
+		policies:  conf.Policies,
+		timeout:   timeout,
+		printHook: logPrinter{logger: logger},
+		options: []func(r *rego.Rego){
+			RFMimetypeDetect,
+			RFResourceDownload,
+			rfMimetypeExtensions,
 		},
-		nil
+	}, nil
 }
 
 // Evaluate evaluates the opa policies and returns the result.
 func (o OPA) Evaluate(ctx context.Context, qs string, env engine.Environment) (bool, error) {
 	ctx, cancel := context.WithTimeout(ctx, o.timeout)
 	defer cancel()
 
-	customFns := []func(r *rego.Rego){
-		RFResourceDownload,
-		RFMimetypeDetect,
-		RFMimetypeExtensions,
-	}
-
 	q, err := rego.New(
 		append([]func(r *rego.Rego){
 			rego.Query(qs),
 			rego.Load(o.policies, nil),
 			rego.EnablePrintStatements(true),
 			rego.PrintHook(o.printHook),
-		}, customFns...)...,
+		}, o.options...)...,
 	).PrepareForEval(ctx)
 	if err != nil {
 		return false, err

services/policies/pkg/engine/opa/rf_mimetype.go

@@ -1,46 +1,81 @@
 package opa
 
 import (
-	"log"
+	"bufio"
+	"io"
 	"mime"
 	"strings"
 
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/open-policy-agent/opa/ast"
 	"github.com/open-policy-agent/opa/rego"
 	"github.com/open-policy-agent/opa/types"
-	"github.com/rakyll/magicmime"
 )
 
-var RFMimetypeExtensions = rego.Function1(
-	&rego.Function{
-		Name:             "ocis.mimetype.extensions",
-		Decl:             types.NewFunction(types.Args(types.S), types.A),
-		Memoize:          true,
-		Nondeterministic: true,
-	},
-	func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
-		var mt string
+// RFMimetypeExtensions extends the rego dictionary with the possibility of mapping mimetypes to file extensions.
+// Be careful calling this multiple times with individual readers, the mime store is global,
+// which results in one global store which holds all known mimetype mappings at once.
+//
+// Rego: `ocis.mimetype.extensions("application/pdf")`
+// Result `[.pdf]`
+func RFMimetypeExtensions(f io.Reader) (func(*rego.Rego), error) {
+	if f != nil {
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			fields := strings.Fields(scanner.Text())
+			if len(fields) <= 1 || fields[0][0] == '#' {
+				continue
+			}
+			mimeType := fields[0]
+			for _, ext := range fields[1:] {
+				if ext[0] == '#' {
+					break
+				}
+				if err := mime.AddExtensionType("."+ext, mimeType); err != nil {
+					return nil, err
+				}
 
-		if err := ast.As(a.Value, &mt); err != nil {
-			return nil, err
+			}
 		}
-		if err := mime.AddExtensionType(".oform", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); err != nil {
-			return nil, err
-		}
-		detectedExtensions, err := mime.ExtensionsByType(mt)
-		if err != nil {
-			return nil, err
+		if err := scanner.Err(); err != nil {
+			panic(err)
 		}
+	}
 
-		var mimeTerms []*ast.Term
-		for _, extension := range detectedExtensions {
-			mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
-		}
+	return rego.Function1(
+		&rego.Function{
+			Name:             "ocis.mimetype.extensions",
+			Decl:             types.NewFunction(types.Args(types.S), types.A),
+			Memoize:          true,
+			Nondeterministic: true,
+		},
+		func(_ rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+			var mt string
 
-		return ast.ArrayTerm(mimeTerms...), nil
-	},
-)
+			if err := ast.As(a.Value, &mt); err != nil {
+				return nil, err
+			}
+
+			detectedExtensions, err := mime.ExtensionsByType(mt)
+			if err != nil {
+				return nil, err
+			}
+
+			var mimeTerms []*ast.Term
+			for _, extension := range detectedExtensions {
+				mimeTerms = append(mimeTerms, ast.NewTerm(ast.String(extension)))
+			}
+
+			return ast.ArrayTerm(mimeTerms...), nil
+		},
+	), nil
+}
 
+// RFMimetypeDetect extends the rego dictionary with the possibility to detect mimetypes.
+// Be careful, the list of known mimetypes is limited.
+//
+// Rego: `ocis.mimetype.extensions(".txt")`
+// Result `text/plain`
 var RFMimetypeDetect = rego.Function1(
 	&rego.Function{
 		Name:             "ocis.mimetype.detect",
@@ -55,14 +90,8 @@ var RFMimetypeDetect = rego.Function1(
 			return nil, err
 		}
 
-		if err := magicmime.Open(magicmime.MAGIC_MIME_TYPE | magicmime.MAGIC_SYMLINK | magicmime.MAGIC_ERROR); err != nil {
-			log.Fatal(err)
-		}
-		defer magicmime.Close()
-		mimetype, err := magicmime.TypeByBuffer(body)
-		if err != nil {
-			return nil, err
-		}
+		mimetype := mimetype.Detect(body).String()
+
 		return ast.StringTerm(strings.Split(mimetype, ";")[0]), nil
 	},
 )

services/policies/pkg/engine/opa/rf_mimetype_test.go

@@ -2,6 +2,8 @@ package opa_test
 
 import (
 	"context"
+	"io"
+	"strings"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -19,13 +21,37 @@ var _ = Describe("opa ocis mimetype functions", func() {
 			Expect(rs[0].Expressions[0].String()).To(Equal("text/plain"))
 		})
 	})
+	Describe("ocis.mimetype.extensions", func() {
+		DescribeTable("resolves extensions by mimetype",
+			func(mimetype string, expectations []string, f io.Reader) {
+				rfMimetypeExtensions, err := opa.RFMimetypeExtensions(f)
+				Expect(err).ToNot(HaveOccurred())
 
-	Describe("ocis.mimetype.extension_for_mimetype", func() {
-		It("provides matching extensions", func() {
-			r := rego.New(rego.Query(`ocis.mimetype.extensions("application/pdf")`), opa.RFMimetypeExtensions)
-			rs, err := r.Eval(context.Background())
-			Expect(err).ToNot(HaveOccurred())
-			Expect(rs[0].Expressions[0].String()).To(Equal("[.pdf]"))
-		})
+				r := rego.New(rego.Query(`ocis.mimetype.extensions("`+mimetype+`")`), rfMimetypeExtensions)
+				rs, err := r.Eval(context.Background())
+				Expect(err).ToNot(HaveOccurred())
+
+				got := rs[0].Expressions[0].String()
+
+				if len(expectations) == 0 {
+					Expect(got).To(Equal("[]"))
+				}
+
+				for i, expectation := range expectations {
+					if i+1 != len(expectations) {
+						expectation += " "
+					}
+
+					Expect(string(got[0])).To(Equal("["))
+					Expect(strings.Contains(got, expectation)).To(BeTrue())
+					Expect(string(got[len(got)-1])).To(Equal("]"))
+				}
+			},
+			Entry("With default mimetype", "application/pdf", []string{".pdf"}, nil),
+			Entry("With unknown mimetype", "ocis/with.custom.mt", []string{}, nil),
+			Entry("With custom mimetype", "ocis/with.custom.mt", []string{".with.custom.mt"}, strings.NewReader("ocis/with.custom.mt    with.custom.mt")),
+			Entry("With multiple custom mimetypes", "ocis/with.multiple.custom.mt", []string{".with.multiple.custom.1.mt", ".with.multiple.custom.2.mt"}, strings.NewReader("ocis/with.multiple.custom.mt                                with.multiple.custom.1.mt with.multiple.custom.2.mt")),
+			Entry("With custom ignored mimetype", "ocis/with.multiple.custom.ignored.mt", []string{}, strings.NewReader("#ocis/with.multiple.custom.ignored.mt with.multiple.custom.ignored.mt")),
+		)
 	})
 })

services/policies/pkg/engine/opa/rf_resource.go

@@ -11,6 +11,10 @@ import (
 	"github.com/open-policy-agent/opa/types"
 )
 
+// RFResourceDownload extends the rego dictionary with the possibility to download oCis resources.
+//
+// Rego: `ocis.resource.download("ocis/path/0034892347349827")`
+// Result: bytes
 var RFResourceDownload = rego.Function1(
 	&rego.Function{
 		Name:             "ocis.resource.download",

vendor/modules.txt

@@ -1565,9 +1565,6 @@ github.com/prometheus/statsd_exporter/pkg/mapper/fsm
 # github.com/r3labs/sse/v2 v2.10.0
 ## explicit; go 1.13
 github.com/r3labs/sse/v2
-# github.com/rakyll/magicmime v0.1.0
-## explicit
-github.com/rakyll/magicmime
 # github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0
 ## explicit
 github.com/rcrowley/go-metrics