Introduce new format pattern + port json processing

cmd/packages.go

@@ -6,6 +6,8 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/anchore/syft/syft/format"
+
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/anchore"
@@ -48,7 +50,7 @@ const (
 )
 
 var (
-	packagesPresenterOpt packages.PresenterOption
+	packagesPresenterOpt format.Option
 	packagesCmd          = &cobra.Command{
 		Use:   "packages [SOURCE]",
 		Short: "Generate a package SBOM",
@@ -62,8 +64,8 @@ var (
 		SilenceErrors: true,
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			// set the presenter
-			presenterOption := packages.ParsePresenterOption(appConfig.Output)
-			if presenterOption == packages.UnknownPresenterOption {
+			presenterOption := format.ParseOption(appConfig.Output)
+			if presenterOption == format.UnknownFormatOption {
 				return fmt.Errorf("bad --output value '%s'", appConfig.Output)
 			}
 			packagesPresenterOpt = presenterOption
@@ -100,8 +102,8 @@ func setPackageFlags(flags *pflag.FlagSet) {
 		fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
 
 	flags.StringP(
-		"output", "o", string(packages.TablePresenterOption),
-		fmt.Sprintf("report output formatter, options=%v", packages.AllPresenters),
+		"output", "o", string(format.TableOption),
+		fmt.Sprintf("report output formatter, options=%v", format.AllPresenters),
 	)
 
 	flags.StringP(

internal/anchore/import_package_sbom.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/anchore/syft/internal/presenter/packages"
+	"github.com/anchore/syft/internal/formats/syftjson"
 
 	"github.com/wagoodman/go-progress"
 
@@ -26,8 +26,8 @@ type packageSBOMImportAPI interface {
 
 func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) (*external.ImagePackageManifest, error) {
 	var buf bytes.Buffer
-	pres := packages.NewJSONPresenter(catalog, s, d, scope)
-	err := pres.Present(&buf)
+
+	err := syftjson.Format().Presenter(catalog, &s, d, scope).Present(&buf)
 	if err != nil {
 		return nil, fmt.Errorf("unable to serialize results: %w", err)
 	}

internal/anchore/import_package_sbom_test.go

@@ -9,18 +9,15 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/anchore/syft/internal/presenter/packages"
-
-	"github.com/wagoodman/go-progress"
-
-	"github.com/anchore/syft/syft/distro"
-
-	"github.com/docker/docker/pkg/ioutils"
-
 	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/formats/syftjson"
+	syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model"
+	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
+	"github.com/docker/docker/pkg/ioutils"
 	"github.com/go-test/deep"
+	"github.com/wagoodman/go-progress"
 )
 
 func must(c pkg.CPE, e error) pkg.CPE {
@@ -88,19 +85,19 @@ func TestPackageSbomToModel(t *testing.T) {
 	}
 
 	var buf bytes.Buffer
-	pres := packages.NewJSONPresenter(c, m, &d, source.AllLayersScope)
+	pres := syftjson.Format().Presenter(c, &m, &d, source.AllLayersScope)
 	if err := pres.Present(&buf); err != nil {
 		t.Fatalf("unable to get expected json: %+v", err)
 	}
 
 	// unmarshal expected result
-	var expectedDoc packages.JSONDocument
+	var expectedDoc syftjsonModel.Document
 	if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil {
 		t.Fatalf("unable to parse json doc: %+v", err)
 	}
 
 	// unmarshal actual result
-	var actualDoc packages.JSONDocument
+	var actualDoc syftjsonModel.Document
 	if err := json.Unmarshal(modelJSON, &actualDoc); err != nil {
 		t.Fatalf("unable to parse json doc: %+v", err)
 	}

internal/presenter/packages/utils_test.go → internal/formats/common/testutils/utils.go

@@ -1,23 +1,24 @@
-package packages
+package testutils
 
 import (
 	"bytes"
 	"testing"
 
+	"github.com/anchore/syft/syft/presenter"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/presenter"
 	"github.com/anchore/syft/syft/source"
 	"github.com/sergi/go-diff/diffmatchpatch"
 	"github.com/stretchr/testify/assert"
 )
 
 type redactor func(s []byte) []byte
 
-func assertPresenterAgainstGoldenImageSnapshot(t *testing.T, pres presenter.Presenter, testImage string, updateSnapshot bool, redactors ...redactor) {
+func AssertPresenterAgainstGoldenImageSnapshot(t *testing.T, pres presenter.Presenter, testImage string, updateSnapshot bool, redactors ...redactor) {
 	var buffer bytes.Buffer
 
 	// grab the latest image contents and persist
@@ -50,7 +51,7 @@ func assertPresenterAgainstGoldenImageSnapshot(t *testing.T, pres presenter.Pres
 	}
 }
 
-func assertPresenterAgainstGoldenSnapshot(t *testing.T, pres presenter.Presenter, updateSnapshot bool, redactors ...redactor) {
+func AssertPresenterAgainstGoldenSnapshot(t *testing.T, pres presenter.Presenter, updateSnapshot bool, redactors ...redactor) {
 	var buffer bytes.Buffer
 
 	err := pres.Present(&buffer)
@@ -77,7 +78,7 @@ func assertPresenterAgainstGoldenSnapshot(t *testing.T, pres presenter.Presenter
 	}
 }
 
-func presenterImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.Metadata, *distro.Distro) {
+func ImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.Metadata, *distro.Distro) {
 	t.Helper()
 	catalog := pkg.NewCatalog()
 	img := imagetest.GetGoldenFixtureImage(t, testImage)
@@ -104,7 +105,7 @@ func presenterImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.M
 		},
 		PURL: "a-purl-1",
 		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+			pkg.MustCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
 		},
 	})
 	catalog.Add(pkg.Package{
@@ -123,7 +124,7 @@ func presenterImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.M
 		},
 		PURL: "a-purl-2",
 		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
+			pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
 		},
 	})
 
@@ -139,7 +140,7 @@ func presenterImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.M
 	return catalog, src.Metadata, &dist
 }
 
-func presenterDirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *distro.Distro) {
+func DirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *distro.Distro) {
 	catalog := pkg.NewCatalog()
 
 	// populate catalog with test data
@@ -160,13 +161,13 @@ func presenterDirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *dist
 			Version: "1.0.1",
 			Files: []pkg.PythonFileRecord{
 				{
-					Path: "/some/path/pkg1/depedencies/foo",
+					Path: "/some/path/pkg1/dependencies/foo",
 				},
 			},
 		},
 		PURL: "a-purl-2",
 		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
+			pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
 		},
 	})
 	catalog.Add(pkg.Package{
@@ -185,7 +186,7 @@ func presenterDirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *dist
 		},
 		PURL: "a-purl-2",
 		CPEs: []pkg.CPE{
-			must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
+			pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
 		},
 	})
 

internal/formats/formats.go

@@ -0,0 +1,34 @@
+package formats
+
+import (
+	"bytes"
+
+	"github.com/anchore/syft/internal/formats/syftjson"
+	"github.com/anchore/syft/syft/format"
+)
+
+// TODO: eventually this is the source of truth for all formatters
+func All() []format.Format {
+	return []format.Format{
+		syftjson.Format(),
+	}
+}
+
+func Identify(by []byte) (*format.Format, error) {
+	for _, f := range All() {
+		if err := f.Validate(bytes.NewReader(by)); err != nil {
+			continue
+		}
+		return &f, nil
+	}
+	return nil, nil
+}
+
+func ByOption(option format.Option) *format.Format {
+	for _, f := range All() {
+		if f.Option == option {
+			return &f
+		}
+	}
+	return nil
+}

internal/formats/formats_test.go

@@ -0,0 +1,34 @@
+package formats
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/format"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIdentify(t *testing.T) {
+	tests := []struct {
+		fixture  string
+		expected format.Option
+	}{
+		{
+			fixture:  "test-fixtures/alpine-syft.json",
+			expected: format.JSONOption,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			f, err := os.Open(test.fixture)
+			assert.NoError(t, err)
+			by, err := io.ReadAll(f)
+			assert.NoError(t, err)
+			frmt, err := Identify(by)
+			assert.NoError(t, err)
+			assert.NotNil(t, frmt)
+			assert.Equal(t, test.expected, frmt.Option)
+		})
+	}
+}

internal/formats/syftjson/decoder.go

@@ -0,0 +1,24 @@
+package syftjson
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/anchore/syft/internal/formats/syftjson/model"
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func decoder(reader io.Reader) (*pkg.Catalog, *source.Metadata, *distro.Distro, source.Scope, error) {
+	dec := json.NewDecoder(reader)
+
+	var doc model.Document
+	err := dec.Decode(&doc)
+	if err != nil {
+		return nil, nil, nil, source.UnknownScope, fmt.Errorf("unable to decode syft-json: %w", err)
+	}
+
+	return toSyftModel(doc)
+}

internal/formats/syftjson/decoder_test.go

@@ -0,0 +1,52 @@
+package syftjson
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/anchore/syft/syft/source"
+
+	"github.com/anchore/syft/internal/formats/common/testutils"
+	"github.com/go-test/deep"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncodeDecodeCycle(t *testing.T) {
+	testImage := "image-simple"
+	originalCatalog, originalMetadata, _ := testutils.ImageInput(t, testImage)
+
+	var buf bytes.Buffer
+	assert.NoError(t, encoder(&buf, originalCatalog, &originalMetadata, nil, source.SquashedScope))
+
+	actualCatalog, actualMetadata, _, _, err := decoder(bytes.NewReader(buf.Bytes()))
+	assert.NoError(t, err)
+
+	for _, d := range deep.Equal(originalMetadata, *actualMetadata) {
+		t.Errorf("metadata difference: %+v", d)
+	}
+
+	actualPackages := actualCatalog.Sorted()
+	for idx, p := range originalCatalog.Sorted() {
+		if !assert.Equal(t, p.Name, actualPackages[idx].Name) {
+			t.Errorf("different package at idx=%d: %s vs %s", idx, p.Name, actualPackages[idx].Name)
+			continue
+		}
+
+		// ids will never be equal
+		p.ID = ""
+		actualPackages[idx].ID = ""
+
+		for _, d := range deep.Equal(*p, *actualPackages[idx]) {
+			if strings.Contains(d, ".VirtualPath: ") {
+				// location.Virtual path is not exposed in the json output
+				continue
+			}
+			if strings.HasSuffix(d, "<nil slice> != []") {
+				// semantically the same
+				continue
+			}
+			t.Errorf("package difference (%s): %+v", p.Name, d)
+		}
+	}
+}

internal/formats/syftjson/encoder.go

@@ -0,0 +1,23 @@
+package syftjson
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/anchore/syft/syft/distro"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func encoder(output io.Writer, catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro, scope source.Scope) error {
+	// TODO: application config not available yet
+	doc := ToFormatModel(catalog, srcMetadata, d, scope, nil)
+
+	enc := json.NewEncoder(output)
+	// prevent > and < from being escaped in the payload
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", " ")
+
+	return enc.Encode(&doc)
+}