Validate content of folder items

code/go/Makefile

@@ -8,9 +8,11 @@ update:
 
 check: lint check-spec 
 
+# "yamlschema" directory has been excluded from linting, because it contains implementations of gojsonschema interfaces
+# which are not compliant with linter rules. The golint tool doesn't support ignore comments.
 lint:
 	@go get -u golang.org/x/lint/golint
-	@go list ./... | xargs -n 1 golint -set_exit_status
+	@go list ./... | grep -v yamlschema | xargs -n 1 golint -set_exit_status
 
 # Checks that the spec is up-to-date
 check-spec:

code/go/go.mod

@@ -8,7 +8,9 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rakyll/statik v0.1.7
 	github.com/stretchr/testify v1.6.1
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415
+	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
-	golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 // indirect
+	golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )

code/go/go.sum

@@ -11,9 +11,17 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
 github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -24,6 +32,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -35,9 +44,16 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuH
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06 h1:ChBCbOHeLqK+j+znGPlWCcvx/t2PdxmyPBheVZxXbcc=
 golang.org/x/tools v0.0.0-20200826040757-bc8aaaa29e06/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 h1:r3P/5xOq/dK1991B65Oy6E1fRF/2d/fSYZJ/fXGVfJc=
+golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200828161849-5deb26317202 h1:DrWbY9UUFi/sl/3HkNVoBjDbGfIPZZfgoGsGxOL1EU8=
+golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65 h1:DajXNh69ob79PCQz1N7OHxmqq6ASZC5xAnJJWIQGR6I=
+golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

code/go/internal/validator/common_spec.go

@@ -0,0 +1,31 @@
+package validator
+
+import (
+	"github.com/creasty/defaults"
+	"github.com/pkg/errors"
+)
+
+type commonSpec struct {
+	AdditionalContents bool                   `yaml:"additionalContents"`
+	Content            map[string]interface{} `yaml:"content"`
+	Contents           []folderItemSpec       `yaml:"contents"`
+}
+
+func setDefaultValues(spec *commonSpec) error {
+	err := defaults.Set(spec)
+	if err != nil {
+		return errors.Wrap(err, "could not set default values")
+	}
+
+	if len(spec.Contents) == 0 {
+		return nil
+	}
+
+	for i := range spec.Contents {
+		err = setDefaultValues(&spec.Contents[i].commonSpec)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

code/go/internal/validator/folder_item_spec.go

@@ -0,0 +1,125 @@
+package validator
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/elastic/package-spec/code/go/internal/yamlschema"
+	"github.com/xeipuuv/gojsonschema"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+type folderItemSpec struct {
+	Description      string `yaml:"description"`
+	ItemType         string `yaml:"type"`
+	ContentMediaType string `yaml:"contentMediaType"`
+	Name             string `yaml:"name"`
+	Pattern          string `yaml:"pattern"`
+	Required         bool   `yaml:"required"`
+	Ref              string `yaml:"$ref"`
+	Visibility       string `yaml:"visibility" default:"public"`
+	commonSpec       `yaml:",inline"`
+}
+
+func (s *folderItemSpec) matchingFileExists(files []os.FileInfo) (bool, error) {
+	if s.Name != "" {
+		for _, file := range files {
+			if file.Name() == s.Name {
+				return s.isSameType(file), nil
+			}
+		}
+	} else if s.Pattern != "" {
+		for _, file := range files {
+			isMatch, err := regexp.MatchString(s.Pattern, file.Name())
+			if err != nil {
+				return false, errors.Wrap(err, "invalid folder item spec pattern")
+			}
+			if isMatch {
+				return s.isSameType(file), nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
+func (s *folderItemSpec) isSameType(file os.FileInfo) bool {
+	switch s.ItemType {
+	case itemTypeFile:
+		return !file.IsDir()
+	case itemTypeFolder:
+		return file.IsDir()
+	}
+
+	return false
+}
+
+func (s *folderItemSpec) validate(fs http.FileSystem, folderSpecPath string, itemPath string) ValidationErrors {
+	var schemaLoader gojsonschema.JSONLoader
+	if s.Ref != "" {
+		schemaPath := filepath.Join(filepath.Dir(folderSpecPath), s.Ref)
+		schemaLoader = yamlschema.NewReferenceLoaderFileSystem("file://"+schemaPath, fs)
+	} else if s.Content != nil {
+		schemaLoader = yamlschema.NewRawLoaderFileSystem(s.Content, fs)
+	} else {
+		return nil // item's schema is not defined
+	}
+
+	// loading item content
+	itemData, err := loadItemContent(itemPath, s.ContentMediaType)
+	if err != nil {
+		return ValidationErrors{errors.Wrapf(err, "loading item content failed (path %s)", itemPath)}
+	}
+
+	// validation with schema
+	documentLoader := gojsonschema.NewBytesLoader(itemData)
+	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
+	if err != nil {
+		return ValidationErrors{err}
+	}
+
+	if result.Valid() {
+		return nil // item content is valid according to the loaded schema
+	}
+
+	var errs ValidationErrors
+	for _, re := range result.Errors() {
+		errs = append(errs, fmt.Errorf("field %s: %s", re.Field(), re.Description()))
+	}
+	return errs
+}
+
+func loadItemContent(itemPath, mediaType string) ([]byte, error) {
+	itemData, err := ioutil.ReadFile(itemPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading item file failed")
+	}
+
+	if len(itemData) == 0 {
+		return nil, errors.New("file is empty")
+	}
+
+	switch mediaType {
+	case "application/x-yaml":
+		var c interface{}
+		err = yaml.Unmarshal(itemData, &c)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", itemPath)
+		}
+
+		itemData, err = json.Marshal(&c)
+		if err != nil {
+			return nil, errors.Wrapf(err, "converting YAML file to JSON failed (path: %s)", itemPath)
+		}
+	case "application/json": // no need to convert the item content
+	default:
+		return nil, fmt.Errorf("unsupported media type (%s)", mediaType)
+	}
+	return itemData, nil
+}

code/go/internal/validator/folder.go → code/go/internal/validator/folder_spec.go

@@ -4,12 +4,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 
-	"github.com/creasty/defaults"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
@@ -28,23 +26,6 @@ type folderSpec struct {
 	commonSpec
 }
 
-type folderItemSpec struct {
-	Description      string `yaml:"description"`
-	ItemType         string `yaml:"type"`
-	ContentMediaType string `yaml:"contentMediaType"`
-	Name             string `yaml:"name"`
-	Pattern          string `yaml:"pattern"`
-	Required         bool   `yaml:"required"`
-	Ref              string `yaml:"$ref"`
-	Visibility         string           `yaml:"visibility" default:"public"`
-	commonSpec       `yaml:",inline"`
-}
-
-type commonSpec struct {
-	AdditionalContents bool             `yaml:"additionalContents"`
-	Contents           []folderItemSpec `yaml:"contents"`
-}
-
 func newFolderSpec(fs http.FileSystem, specPath string) (*folderSpec, error) {
 	specFile, err := fs.Open(specPath)
 	if err != nil {
@@ -151,7 +132,14 @@ func (s *folderSpec) validate(folderPath string) ValidationErrors {
 				errs = append(errs, fmt.Errorf("[%s] is a file but is expected to be a folder", fileName))
 				continue
 			}
-			// TODO: more validation for file item
+
+			itemPath := filepath.Join(folderPath, file.Name())
+			itemValidationErrs := itemSpec.validate(s.fs, s.specPath, itemPath)
+			if itemValidationErrs != nil {
+				for _, ive := range itemValidationErrs {
+					errs = append(errs, errors.Wrapf(ive, "file \"%s\" is invalid", itemPath))
+				}
+			}
 		}
 	}
 
@@ -198,56 +186,4 @@ func (s *folderSpec) findItemSpec(folderItemName string) (*folderItemSpec, error
 
 	// No item spec found
 	return nil, nil
-}
-
-func (s *folderItemSpec) matchingFileExists(files []os.FileInfo) (bool, error) {
-	if s.Name != "" {
-		for _, file := range files {
-			if file.Name() == s.Name {
-				return s.isSameType(file), nil
-			}
-		}
-	} else if s.Pattern != "" {
-		for _, file := range files {
-			isMatch, err := regexp.MatchString(s.Pattern, file.Name())
-			if err != nil {
-				return false, errors.Wrap(err, "invalid folder item spec pattern")
-			}
-			if isMatch {
-				return s.isSameType(file), nil
-			}
-		}
-	}
-
-	return false, nil
-}
-
-func (s *folderItemSpec) isSameType(file os.FileInfo) bool {
-	switch s.ItemType {
-	case itemTypeFile:
-		return !file.IsDir()
-	case itemTypeFolder:
-		return file.IsDir()
-	}
-
-	return false
-}
-
-func setDefaultValues(spec *commonSpec) error {
-	err := defaults.Set(spec)
-	if err != nil {
-		return errors.Wrap(err, "could not set default values")
-	}
-
-	if len(spec.Contents) == 0 {
-		return nil
-	}
-
-	for i := range spec.Contents {
-		err = setDefaultValues(&spec.Contents[i].commonSpec)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
+}

code/go/internal/validator/test/packages/good/dataset/foo/_dev/test/pipeline/test-access-raw.log-expected.json

@@ -1,5 +1,5 @@
 {
-    "1expected": [
+    "expected": [
         {
             "event.category": [
                 "web"

code/go/internal/validator/test/packages/good/dataset/foo/fields/some_fields.yml

@@ -0,0 +1,24 @@
+- name: source
+  title: Source
+  group: 2
+  type: group
+  fields:
+    - name: geo.city_name
+      level: core
+      type: keyword
+      description: City name.
+      ignore_above: 1024
+    - name: geo.location
+      level: core
+      type: geo_point
+      description: Longitude and latitude.
+    - name: geo.region_iso_code
+      level: core
+      type: keyword
+      description: Region ISO code.
+      ignore_above: 1024
+    - name: geo.region_name
+      level: core
+      type: keyword
+      description: Region name.
+      ignore_above: 1024

code/go/internal/validator/test/packages/good/dataset/foo/manifest.yml

@@ -0,0 +1,23 @@
+title: Nginx access logs
+type: logs
+release: experimental
+streams:
+  - input: logfile
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - /var/log/nginx/access.log*
+      - name: server_status_path
+        type: text
+        title: Server Status Path
+        multi: false
+        required: true
+        show_user: false
+        default: /server-status
+    title: Nginx access logs
+    description: Collect Nginx access logs