chore: account for syft package metadata changes (anchore#1423)

Signed-off-by: Alex Goodman <[email protected]>
Signed-off-by: Keith Zantow <[email protected]>
Co-authored-by: Keith Zantow <[email protected]>

Makefile

@@ -259,8 +259,11 @@ compare-test-rpm-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR)
 			$(COMPARE_TEST_IMAGE) \
 			$(TEMP_DIR)
 
-## Code generation targets #################################
-## TODO (cphillips) what does grype have here?
+## Code and data generation targets #################################
+
+.PHONY: generate
+generate:  ## Generate any code or data required by the project
+	cd grype/internal && go generate .
 
 ## Build-related targets #################################
 

go.mod

@@ -60,6 +60,8 @@ require (
 	gorm.io/gorm v1.25.5
 )
 
+require github.com/dave/jennifer v1.7.0
+
 require (
 	cloud.google.com/go v0.110.7 // indirect
 	cloud.google.com/go/compute v1.23.0 // indirect

go.sum

@@ -361,6 +361,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/dave/jennifer v1.7.0 h1:uRbSBH9UTS64yXbh4FrMHfgfY762RD+C7bUPKODpSJE=
+github.com/dave/jennifer v1.7.0/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

grype/db/v4/pkg/resolver/java/resolver_test.go

@@ -53,10 +53,9 @@ func TestResolver_Resolve(t *testing.T) {
 		{
 			name: "both artifact and manifest 1",
 			pkg: grypePkg.Package{
-				Name:         "ABCD",
-				Version:      "1.2.3.4",
-				Language:     "java",
-				MetadataType: "",
+				Name:     "ABCD",
+				Version:  "1.2.3.4",
+				Language: "java",
 				Metadata: grypePkg.JavaMetadata{
 					VirtualPath:   "virtual-path-info",
 					PomArtifactID: "pom-ARTIFACT-ID-info",

grype/db/v5/pkg/resolver/java/resolver_test.go

@@ -53,10 +53,9 @@ func TestResolver_Resolve(t *testing.T) {
 		{
 			name: "both artifact and manifest 1",
 			pkg: grypePkg.Package{
-				Name:         "ABCD",
-				Version:      "1.2.3.4",
-				Language:     "java",
-				MetadataType: "",
+				Name:     "ABCD",
+				Version:  "1.2.3.4",
+				Language: "java",
 				Metadata: grypePkg.JavaMetadata{
 					VirtualPath:   "virtual-path-info",
 					PomArtifactID: "pom-ARTIFACT-ID-info",

grype/internal/generate.go

@@ -0,0 +1,3 @@
+package internal
+
+//go:generate go run ./packagemetadata/generate/main.go

grype/internal/packagemetadata/discover_type_names.go

@@ -0,0 +1,150 @@
+package packagemetadata
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"unicode"
+
+	"github.com/scylladb/go-set/strset"
+)
+
+var metadataExceptions = strset.New(
+	"FileMetadata",
+)
+
+func DiscoverTypeNames() ([]string, error) {
+	root, err := RepoRoot()
+	if err != nil {
+		return nil, err
+	}
+	files, err := filepath.Glob(filepath.Join(root, "grype/pkg/*.go"))
+	if err != nil {
+		return nil, err
+	}
+	return findMetadataDefinitionNames(files...)
+}
+
+func RepoRoot() (string, error) {
+	root, err := exec.Command("git", "rev-parse", "--show-toplevel").Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to find repo root dir: %+v", err)
+	}
+	absRepoRoot, err := filepath.Abs(strings.TrimSpace(string(root)))
+	if err != nil {
+		return "", fmt.Errorf("unable to get abs path to repo root: %w", err)
+	}
+	return absRepoRoot, nil
+}
+
+func findMetadataDefinitionNames(paths ...string) ([]string, error) {
+	names := strset.New()
+	usedNames := strset.New()
+	for _, path := range paths {
+		metadataDefinitions, usedTypeNames, err := findMetadataDefinitionNamesInFile(path)
+		if err != nil {
+			return nil, err
+		}
+
+		// useful for debugging...
+		// fmt.Println(path)
+		// fmt.Println("Defs:", metadataDefinitions)
+		// fmt.Println("Used Types:", usedTypeNames)
+		// fmt.Println()
+
+		names.Add(metadataDefinitions...)
+		usedNames.Add(usedTypeNames...)
+	}
+
+	// any definition that is used within another struct should not be considered a top-level metadata definition
+	names.Remove(usedNames.List()...)
+
+	strNames := names.List()
+	sort.Strings(strNames)
+
+	// note: 3 is a point-in-time gut check. This number could be updated if new metadata definitions are added, but is not required.
+	// it is really intended to catch any major issues with the generation process that would generate, say, 0 definitions.
+	if len(strNames) < 3 {
+		return nil, fmt.Errorf("not enough metadata definitions found (discovered: " + fmt.Sprintf("%d", len(strNames)) + ")")
+	}
+
+	return strNames, nil
+}
+
+func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error) {
+	// set up the parser
+	fs := token.NewFileSet()
+	f, err := parser.ParseFile(fs, path, nil, parser.ParseComments)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var metadataDefinitions []string
+	var usedTypeNames []string
+	for _, decl := range f.Decls {
+		// check if the declaration is a type declaration
+		spec, ok := decl.(*ast.GenDecl)
+		if !ok || spec.Tok != token.TYPE {
+			continue
+		}
+
+		// loop over all types declared in the type declaration
+		for _, typ := range spec.Specs {
+			// check if the type is a struct type
+			spec, ok := typ.(*ast.TypeSpec)
+			if !ok || spec.Type == nil {
+				continue
+			}
+
+			structType, ok := spec.Type.(*ast.StructType)
+			if !ok {
+				continue
+			}
+
+			// check if the struct type ends with "Metadata"
+			name := spec.Name.String()
+
+			// only look for exported types that end with "Metadata"
+			if isMetadataTypeCandidate(name) {
+				// print the full declaration of the struct type
+				metadataDefinitions = append(metadataDefinitions, name)
+				usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
+			}
+		}
+	}
+	return metadataDefinitions, usedTypeNames, nil
+}
+
+func typeNamesUsedInStruct(structType *ast.StructType) []string {
+	// recursively find all type names used in the struct type
+	var names []string
+	for i := range structType.Fields.List {
+		// capture names of all of the types (not field names)
+		ast.Inspect(structType.Fields.List[i].Type, func(n ast.Node) bool {
+			ident, ok := n.(*ast.Ident)
+			if !ok {
+				return true
+			}
+
+			// add the type name to the list
+			names = append(names, ident.Name)
+
+			// continue inspecting
+			return true
+		})
+	}
+
+	return names
+}
+
+func isMetadataTypeCandidate(name string) bool {
+	return len(name) > 0 &&
+		strings.HasSuffix(name, "Metadata") &&
+		unicode.IsUpper(rune(name[0])) && // must be exported
+		!metadataExceptions.Has(name)
+}

grype/internal/packagemetadata/generate/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/dave/jennifer/jen"
+
+	"github.com/anchore/grype/grype/internal/packagemetadata"
+)
+
+// This program is invoked from grype/internal and generates packagemetadata/generated.go
+
+const (
+	pkgImport = "github.com/anchore/grype/grype/pkg"
+	path      = "packagemetadata/generated.go"
+)
+
+func main() {
+	typeNames, err := packagemetadata.DiscoverTypeNames()
+	if err != nil {
+		panic(fmt.Errorf("unable to get all metadata type names: %w", err))
+	}
+
+	fmt.Printf("updating package metadata type list with %+v types\n", len(typeNames))
+
+	f := jen.NewFile("packagemetadata")
+	f.HeaderComment("DO NOT EDIT: generated by grype/internal/packagemetadata/generate/main.go")
+	f.ImportName(pkgImport, "pkg")
+	f.Comment("AllTypes returns a list of all pkg metadata types that grype supports (that are represented in the pkg.Package.Metadata field).")
+
+	f.Func().Id("AllTypes").Params().Index().Any().BlockFunc(func(g *jen.Group) {
+		g.ReturnFunc(func(g *jen.Group) {
+			g.Index().Any().ValuesFunc(func(g *jen.Group) {
+				for _, typeName := range typeNames {
+					g.Qual(pkgImport, typeName).Values()
+				}
+			})
+		})
+	})
+
+	rendered := fmt.Sprintf("%#v", f)
+
+	fh, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		panic(fmt.Errorf("unable to open file: %w", err))
+	}
+	_, err = fh.WriteString(rendered)
+	if err != nil {
+		panic(fmt.Errorf("unable to write file: %w", err))
+	}
+	if err := fh.Close(); err != nil {
+		panic(fmt.Errorf("unable to close file: %w", err))
+	}
+}

grype/internal/packagemetadata/generated.go

@@ -0,0 +1,10 @@
+// DO NOT EDIT: generated by grype/internal/packagemetadata/generate/main.go
+
+package packagemetadata
+
+import "github.com/anchore/grype/grype/pkg"
+
+// AllTypes returns a list of all pkg metadata types that grype supports (that are represented in the pkg.Package.Metadata field).
+func AllTypes() []any {
+	return []any{pkg.GolangBinMetadata{}, pkg.GolangModMetadata{}, pkg.JavaMetadata{}, pkg.RpmMetadata{}}
+}

grype/internal/packagemetadata/names.go

@@ -0,0 +1,81 @@
+package packagemetadata
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/anchore/grype/grype/pkg"
+)
+
+// jsonNameFromType is a map of all known package metadata types to their current JSON name and all previously known aliases.
+// TODO: in the future the metadata type names should match how it is used in syft. However, since the data shapes are
+// not the same it may be important to select different names. This design decision has been deferred, for now
+// the same metadata types that have been used in the past should be used here.
+var jsonNameFromType = map[reflect.Type][]string{
+	reflect.TypeOf(pkg.GolangBinMetadata{}): nameList("GolangBinMetadata"),
+	reflect.TypeOf(pkg.GolangModMetadata{}): nameList("GolangModMetadata"),
+	reflect.TypeOf(pkg.JavaMetadata{}):      nameList("JavaMetadata"),
+	reflect.TypeOf(pkg.RpmMetadata{}):       nameList("RpmMetadata"),
+}
+
+//nolint:unparam
+func nameList(id string, others ...string) []string {
+	names := []string{id}
+	for _, o := range others {
+		names = append(names, expandLegacyNameVariants(o)...)
+	}
+	return names
+}
+
+func expandLegacyNameVariants(name string) []string {
+	candidates := []string{name}
+	if strings.HasSuffix(name, "MetadataType") {
+		candidates = append(candidates, strings.TrimSuffix(name, "Type"))
+	} else if strings.HasSuffix(name, "Metadata") {
+		candidates = append(candidates, name+"Type")
+	}
+	return candidates
+}
+
+func AllTypeNames() []string {
+	names := make([]string, 0)
+	for _, t := range AllTypes() {
+		names = append(names, reflect.TypeOf(t).Name())
+	}
+	return names
+}
+
+func JSONName(metadata any) string {
+	if vs, exists := jsonNameFromType[reflect.TypeOf(metadata)]; exists {
+		return vs[0]
+	}
+	return ""
+}
+
+func ReflectTypeFromJSONName(name string) reflect.Type {
+	name = strings.ToLower(name)
+	for _, t := range sortedTypes(jsonNameFromType) {
+		vs := jsonNameFromType[t]
+		for _, v := range vs {
+			if strings.ToLower(v) == name {
+				return t
+			}
+		}
+	}
+	return nil
+}
+
+func sortedTypes(typeNameMapping map[reflect.Type][]string) []reflect.Type {
+	types := make([]reflect.Type, 0)
+	for t := range typeNameMapping {
+		types = append(types, t)
+	}
+
+	// sort the types by their first JSON name
+	sort.Slice(types, func(i, j int) bool {
+		return typeNameMapping[types[i]][0] < typeNameMapping[types[j]][0]
+	})
+
+	return types
+}