Generate notice file from dependency licences (#1764)

* Generate notice file from dependency licences

* Temporarily ignore lint check

* Add notice file

* Add missing test directory

* Change copyright year range

.golangci.yml

@@ -2,6 +2,7 @@ run:
   deadline: 300s
   skip-dirs:
     - config
+    - hack/licence\-detector
   skip-files:
     - utils/chrono/millis\.test\.go
 linters:

Makefile

@@ -105,10 +105,14 @@ generate: controller-gen
 	$(MAKE) --no-print-directory generate-all-in-one
 	# TODO (sabo): reenable when new tag is cut and can work with the new repo path
 	# $(MAKE) --no-print-directory generate-api-docs
+	$(MAKE) --no-print-directory generate-notice-file
 
 generate-api-docs:
 	@hack/api-docs/build.sh
 
+generate-notice-file:
+	@hack/licence-detector/generate-notice.sh
+
 elastic-operator: generate
 	go build -ldflags "$(GO_LDFLAGS)" -tags='$(GO_TAGS)' -o bin/elastic-operator github.com/elastic/cloud-on-k8s/cmd
 

hack/licence-detector/.gitignore

@@ -0,0 +1,2 @@
+notice.txt
+licence-detector

hack/licence-detector/NOTICE.txt.tmpl

@@ -0,0 +1,34 @@
+{{- define "depInfo" -}}
+{{- range $i, $dep := . }}
+{{ "-" | line }}
+{{ if $dep.Replace -}}
+Module  : {{ $dep.Path }} => {{ $dep.Replace.Path }}
+Version : {{ $dep.Replace.Version }}
+Time    : {{ $dep.Replace.Time }}
+{{- else -}}
+Module  : {{ $dep.Path }}
+Version : {{ $dep.Version }}
+Time    : {{ $dep.Time }}
+{{- end }}
+
+{{ $dep | licenceText }}
+{{ end }}
+{{- end -}}
+
+Copyright 2018-{{ currentYear }} Elasticsearch BV
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
+
+{{ "=" | line }}
+Third party libraries used by the Elastic Cloud on Kubernetes project
+{{ "=" | line }}
+
+{{ template "depInfo" .Direct }}
+
+{{ if .Indirect }}
+{{ "=" | line }}
+Indirect dependencies
+
+{{ template "depInfo" .Indirect }}
+{{ end }}

hack/licence-detector/detector/detector.go

@@ -0,0 +1,134 @@
+package detector
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/karrick/godirwalk"
+)
+
+var errLicenceNotFound = errors.New("failed to detect licence")
+
+type Dependencies struct {
+	Direct   []LicenceInfo
+	Indirect []LicenceInfo
+}
+
+type LicenceInfo struct {
+	Module
+	LicenceFile string
+	Error       error
+}
+
+type Module struct {
+	Path     string     // module path
+	Version  string     // module version
+	Main     bool       // is this the main module?
+	Time     *time.Time // time version was created
+	Indirect bool       // is this module only an indirect dependency of main module?
+	Dir      string     // directory holding files for this module, if any
+	Replace  *Module    // replace directive
+}
+
+func Detect(data io.Reader, includeIndirect bool) (*Dependencies, error) {
+	dependencies, err := parseDependencies(data, includeIndirect)
+	if err != nil {
+		return nil, err
+	}
+
+	err = detectLicences(dependencies)
+	return dependencies, err
+}
+
+func parseDependencies(data io.Reader, includeIndirect bool) (*Dependencies, error) {
+	deps := &Dependencies{}
+	decoder := json.NewDecoder(data)
+	for {
+		var mod Module
+		if err := decoder.Decode(&mod); err != nil {
+			if err == io.EOF {
+				return deps, nil
+			}
+			return deps, fmt.Errorf("failed to parse dependencies: %w", err)
+		}
+
+		if !mod.Main && mod.Dir != "" {
+			if mod.Indirect {
+				if includeIndirect {
+					deps.Indirect = append(deps.Indirect, LicenceInfo{Module: mod})
+				}
+				continue
+			}
+			deps.Direct = append(deps.Direct, LicenceInfo{Module: mod})
+		}
+	}
+}
+
+func detectLicences(deps *Dependencies) error {
+	licenceRegex := buildLicenceRegex()
+	for _, depList := range [][]LicenceInfo{deps.Direct, deps.Indirect} {
+		for i, dep := range depList {
+			srcDir := dep.Dir
+			if dep.Replace != nil {
+				srcDir = dep.Replace.Dir
+			}
+
+			depList[i].LicenceFile, depList[i].Error = findLicenceFile(srcDir, licenceRegex)
+			if depList[i].Error != nil && depList[i].Error != errLicenceNotFound {
+				return fmt.Errorf("unexpected error while finding licence for %s in %s: %w", dep.Path, srcDir, depList[i].Error)
+			}
+		}
+	}
+
+	return nil
+}
+
+func buildLicenceRegex() *regexp.Regexp {
+	// inspired by https://github.com/src-d/go-license-detector/blob/7961dd6009019bc12778175ef7f074ede24bd128/licensedb/internal/investigation.go#L29
+	licenceFileNames := []string{
+		`li[cs]en[cs]es?`,
+		`legal`,
+		`copy(left|right|ing)`,
+		`unlicense`,
+		`l?gpl([-_ v]?)(\d\.?\d)?`,
+		`bsd`,
+		`mit`,
+		`apache`,
+	}
+
+	regexStr := fmt.Sprintf(`^(?i:(%s)(\.(txt|md|rst))?)$`, strings.Join(licenceFileNames, "|"))
+	return regexp.MustCompile(regexStr)
+}
+
+func findLicenceFile(root string, licenceRegex *regexp.Regexp) (string, error) {
+	errStopWalk := errors.New("stop walk")
+	var licenceFile string
+	err := godirwalk.Walk(root, &godirwalk.Options{
+		Callback: func(osPathName string, dirent *godirwalk.Dirent) error {
+			if licenceRegex.MatchString(dirent.Name()) {
+				if dirent.IsDir() {
+					return filepath.SkipDir
+				}
+				licenceFile = osPathName
+				return errStopWalk
+			}
+			return nil
+		},
+		Unsorted: true,
+	})
+
+	if err != nil {
+		if errors.Is(err, errStopWalk) {
+			return licenceFile, nil
+		}
+		return "", err
+	}
+
+	return "", errLicenceNotFound
+}

hack/licence-detector/detector/detector_test.go

@@ -0,0 +1,122 @@
+package detector
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDetect(t *testing.T) {
+	testCases := []struct {
+		name             string
+		includeIndirect  bool
+		wantDependencies *Dependencies
+		wantErr          bool
+	}{
+		{
+			name:            "All",
+			includeIndirect: true,
+			wantDependencies: &Dependencies{
+				Indirect: mkIndirectDeps(),
+				Direct:   mkDirectDeps(),
+			},
+		},
+		{
+			name:            "DirectOnly",
+			includeIndirect: false,
+			wantDependencies: &Dependencies{
+				Direct: mkDirectDeps(),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			f, err := os.Open("testdata/deps.json")
+			require.NoError(t, err)
+			defer f.Close()
+
+			gotDependencies, err := Detect(f, tc.includeIndirect)
+			if tc.wantErr {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, tc.wantDependencies, gotDependencies)
+		})
+	}
+}
+
+func mkIndirectDeps() []LicenceInfo {
+	return []LicenceInfo{
+		{
+			Module: Module{
+				Path:     "github.com/davecgh/go-spew",
+				Version:  "v1.1.0",
+				Time:     mustParseTime("2016-10-29T20:57:26Z"),
+				Indirect: true,
+				Dir:      "testdata/github.com/davecgh/[email protected]",
+			},
+			LicenceFile: "testdata/github.com/davecgh/[email protected]/LICENCE.txt",
+		},
+		{
+			Module: Module{
+				Path:     "github.com/dgryski/go-minhash",
+				Version:  "v0.0.0-20170608043002-7fe510aff544",
+				Time:     mustParseTime("2017-06-08T04:30:02Z"),
+				Indirect: true,
+				Dir:      "testdata/github.com/dgryski/[email protected]",
+			},
+			LicenceFile: "testdata/github.com/dgryski/[email protected]/licence",
+		},
+		{
+			Module: Module{
+				Path:     "github.com/dgryski/go-spooky",
+				Version:  "v0.0.0-20170606183049-ed3d087f40e2",
+				Time:     mustParseTime("2017-06-06T18:30:49Z"),
+				Indirect: true,
+				Dir:      "testdata/github.com/dgryski/[email protected]",
+			},
+			LicenceFile: "testdata/github.com/dgryski/[email protected]/COPYING",
+		},
+	}
+}
+
+func mkDirectDeps() []LicenceInfo {
+	return []LicenceInfo{
+		{
+			Module: Module{
+				Path:    "github.com/ekzhu/minhash-lsh",
+				Version: "v0.0.0-20171225071031-5c06ee8586a1",
+				Time:    mustParseTime("2017-12-25T07:10:31Z"),
+				Dir:     "testdata/github.com/ekzhu/[email protected]",
+			},
+			Error: errLicenceNotFound,
+		},
+		{
+			Module: Module{
+				Path:    "gopkg.in/russross/blackfriday.v2",
+				Version: "v2.0.1",
+				Replace: &Module{
+					Path:    "github.com/russross/blackfriday/v2",
+					Version: "v2.0.1",
+					Time:    mustParseTime("2018-09-20T17:16:15Z"),
+					Dir:     "testdata/github.com/russross/blackfriday/[email protected]",
+				},
+				Dir: "testdata/github.com/russross/blackfriday/[email protected]",
+			},
+			LicenceFile: "testdata/github.com/russross/blackfriday/[email protected]/LICENSE.rst",
+		},
+	}
+}
+
+func mustParseTime(value string) *time.Time {
+	t, err := time.Parse(time.RFC3339, value)
+	if err != nil {
+		panic(err)
+	}
+	return &t
+}

hack/licence-detector/detector/testdata/deps.json

@@ -0,0 +1,51 @@
+{
+	"Path": "github.com/charith-elastic/licence-detector",
+	"Main": true,
+	"Dir": "testdata/github.com/charith-elastic/license-detector",
+	"GoMod": "testdata/github.com/charith-elastic/license-detector/go.mod",
+	"GoVersion": "1.13"
+}
+{
+	"Path": "github.com/davecgh/go-spew",
+	"Version": "v1.1.0",
+	"Time": "2016-10-29T20:57:26Z",
+	"Indirect": true,
+	"Dir": "testdata/github.com/davecgh/[email protected]",
+	"GoMod": "testdata/cache/download/github.com/davecgh/go-spew/@v/v1.1.0.mod"
+}
+{
+	"Path": "github.com/dgryski/go-minhash",
+	"Version": "v0.0.0-20170608043002-7fe510aff544",
+	"Time": "2017-06-08T04:30:02Z",
+	"Indirect": true,
+	"Dir": "testdata/github.com/dgryski/[email protected]",
+	"GoMod": "testdata/cache/download/github.com/dgryski/go-minhash/@v/v0.0.0-20170608043002-7fe510aff544.mod"
+}
+{
+	"Path": "github.com/dgryski/go-spooky",
+	"Version": "v0.0.0-20170606183049-ed3d087f40e2",
+	"Time": "2017-06-06T18:30:49Z",
+	"Indirect": true,
+	"Dir": "testdata/github.com/dgryski/[email protected]",
+	"GoMod": "testdata/cache/download/github.com/dgryski/go-spooky/@v/v0.0.0-20170606183049-ed3d087f40e2.mod"
+}
+{
+	"Path": "github.com/ekzhu/minhash-lsh",
+	"Version": "v0.0.0-20171225071031-5c06ee8586a1",
+	"Time": "2017-12-25T07:10:31Z",
+	"Dir": "testdata/github.com/ekzhu/[email protected]",
+	"GoMod": "testdata/cache/download/github.com/ekzhu/minhash-lsh/@v/v0.0.0-20171225071031-5c06ee8586a1.mod"
+}
+{
+	"Path": "gopkg.in/russross/blackfriday.v2",
+	"Version": "v2.0.1",
+	"Replace": {
+		"Path": "github.com/russross/blackfriday/v2",
+		"Version": "v2.0.1",
+		"Time": "2018-09-20T17:16:15Z",
+		"Dir": "testdata/github.com/russross/blackfriday/[email protected]",
+		"GoMod": "testdata/cache/download/github.com/russross/blackfriday/v2/@v/v2.0.1.mod"
+	},
+	"Dir": "testdata/github.com/russross/blackfriday/[email protected]",
+	"GoMod": "testdata/github.com/russross/blackfriday/[email protected]/go.mod"
+}

hack/licence-detector/detector/testdata/github.com/davecgh/go-spew@v1.1.0/LICENCE.txt

@@ -0,0 +1 @@
+LICENCE

hack/licence-detector/detector/testdata/github.com/dgryski/go-minhash@v0.0.0-20170608043002-7fe510aff544/licence

@@ -0,0 +1 @@
+licence

hack/licence-detector/detector/testdata/github.com/dgryski/go-spooky@v0.0.0-20170606183049-ed3d087f40e2/COPYING

@@ -0,0 +1 @@
+licence

hack/licence-detector/detector/testdata/github.com/ekzhu/minhash-lsh@v0.0.0-20171225071031-5c06ee8586a1/some_file.txt

@@ -0,0 +1 @@
+some text