internal/govulncheck: use a streaming JSON format

Instead of writing the JSON all at once, use a streaming JSON output so
that we can provide progress updates to the user.

The JSON output also now includes config information.

Delete json_vulns.ct, since the tool_version is now encoded and flaky.
Test for exit codes will be added in a later CL.

Change-Id: I26ab7cf99799f3df4426d65c2b78c07877110409
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/477429
Reviewed-by: Ian Cottrell <[email protected]>
Run-TryBot: Julie Qiu <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Julie Qiu <[email protected]>

internal/govulncheck/json.go

@@ -6,47 +6,68 @@ package govulncheck
 
 import (
 	"encoding/json"
-	"fmt"
 
 	"io"
 
 	"golang.org/x/vuln/internal/result"
 )
 
 type jsonHandler struct {
-	w     io.Writer
-	vulns []*result.Vuln
+	enc *json.Encoder
 }
 
 // NewJSONHandler returns a handler that writes govulncheck output as json.
-func NewJSONHandler(to io.Writer) Handler {
-	return &jsonHandler{w: to}
+func NewJSONHandler(w io.Writer) Handler {
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "  ")
+	return &jsonHandler{enc: enc}
+}
+
+// HandleJSON reads the json from the supplied stream and hands the decoded
+// output to the handler.
+func HandleJSON(from io.Reader, to Handler) error {
+	dec := json.NewDecoder(from)
+	for dec.More() {
+		msg := result.Message{}
+		// decode the next message in the stream
+		if err := dec.Decode(&msg); err != nil {
+			return err
+		}
+		// dispatch the message
+		//TODO: should we verify only one field was set?
+		var err error
+		if msg.Preamble != nil {
+			err = to.Preamble(msg.Preamble)
+		}
+		if msg.Vulnerability != nil {
+			err = to.Vulnerability(msg.Vulnerability)
+		}
+		if msg.Progress != "" {
+			err = to.Progress(msg.Progress)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 // Flush writes all vulnerabilities in JSON format.
 func (o *jsonHandler) Flush() error {
-	b, err := json.MarshalIndent(o.vulns, "", " ")
-	o.vulns = nil
-	if err != nil {
-		return err
-	}
-	_, err = o.w.Write(b)
-	fmt.Fprintln(o.w)
-	return err
+	return nil
 }
 
 // Vulnerability gathers vulnerabilities to be written.
 func (o *jsonHandler) Vulnerability(vuln *result.Vuln) error {
-	o.vulns = append(o.vulns, vuln)
-	return nil
+	return o.enc.Encode(result.Message{Vulnerability: vuln})
 }
 
 // Preamble does not do anything in JSON mode.
 func (o *jsonHandler) Preamble(preamble *result.Preamble) error {
-	return nil
+	return o.enc.Encode(result.Message{Preamble: preamble})
 }
 
 // Progress does not do anything in JSON mode.
 func (o *jsonHandler) Progress(msg string) error {
-	return nil
+	return o.enc.Encode(result.Message{Progress: msg})
 }

internal/govulncheck/print_test.go

@@ -128,7 +128,6 @@ var testVuln2 = &osv.Entry{
 	}}}
 
 func TestPrintTextNoVulns(t *testing.T) {
-	testdata := os.DirFS("testdata")
 	preamble := &result.Preamble{
 		Analysis: result.AnalysisSource,
 		Mode:     result.ModeCompact,
@@ -145,11 +144,9 @@ func TestPrintTextNoVulns(t *testing.T) {
 			},
 		},
 	}
-	want, _ := fs.ReadFile(testdata, "no_vulns.txt")
-	testPrint(t, preamble, r, string(want))
+	testPrint(t, preamble, r, "no_vulns")
 }
 func TestPrintTextSource(t *testing.T) {
-	testdata := os.DirFS("testdata")
 	preamble := &result.Preamble{
 		Analysis: result.AnalysisSource,
 		Mode:     result.ModeCompact,
@@ -184,11 +181,10 @@ func TestPrintTextSource(t *testing.T) {
 				},
 			},
 		}}
-	want, _ := fs.ReadFile(testdata, "source.txt")
-	testPrint(t, preamble, r, string(want))
+
+	testPrint(t, preamble, r, "source")
 }
 func TestPrintTextBinary(t *testing.T) {
-	testdata := os.DirFS("testdata")
 	preamble := &result.Preamble{
 		Analysis: result.AnalysisBinary,
 		Mode:     result.ModeCompact,
@@ -220,11 +216,9 @@ func TestPrintTextBinary(t *testing.T) {
 				},
 			},
 		}}
-	want, _ := fs.ReadFile(testdata, "binary.txt")
-	testPrint(t, preamble, r, string(want))
+	testPrint(t, preamble, r, "binary")
 }
 func TestPrintTextMultiModuleAndStacks(t *testing.T) {
-	testdata := os.DirFS("testdata")
 	preamble := &result.Preamble{
 		Analysis: result.AnalysisSource,
 		Mode:     result.ModeCompact,
@@ -258,13 +252,27 @@ func TestPrintTextMultiModuleAndStacks(t *testing.T) {
 				},
 			},
 		}}
-	want, _ := fs.ReadFile(testdata, "multi_stacks.txt")
-	testPrint(t, preamble, r, string(want))
+
+	testPrint(t, preamble, r, "multi_stacks")
 }
 
-func testPrint(t *testing.T, preamble *result.Preamble, vulns []*result.Vuln, want string) {
-	got := new(strings.Builder)
-	output := NewTextHandler(got, preamble)
+func testPrint(t *testing.T, preamble *result.Preamble, vulns []*result.Vuln, name string) {
+	testdata := os.DirFS("testdata")
+	wantText, _ := fs.ReadFile(testdata, name+".txt")
+	wantJSON, _ := fs.ReadFile(testdata, name+".json")
+	got := &strings.Builder{}
+	testRunHandler(t, preamble, vulns, NewTextHandler(got, preamble))
+	if diff := cmp.Diff(string(wantText), got.String()); diff != "" {
+		t.Errorf("Readable mismatch (-want, +got):\n%s", diff)
+	}
+	got.Reset()
+	testRunHandler(t, preamble, vulns, NewJSONHandler(got))
+	if diff := cmp.Diff(string(wantJSON), got.String()); diff != "" {
+		t.Errorf("JSON mismatch (-want, +got):\n%s", diff)
+	}
+}
+
+func testRunHandler(t *testing.T, preamble *result.Preamble, vulns []*result.Vuln, output Handler) {
 	output.Preamble(preamble)
 	for _, v := range vulns {
 		if err := output.Vulnerability(v); err != nil {
@@ -274,7 +282,4 @@ func testPrint(t *testing.T, preamble *result.Preamble, vulns []*result.Vuln, wa
 	if err := output.Flush(); err != nil {
 		t.Fatal(err)
 	}
-	if diff := cmp.Diff(want, got.String()); diff != "" {
-		t.Fatalf("mismatch (-want, +got):\n%s", diff)
-	}
 }

internal/govulncheck/testdata/binary.json

@@ -0,0 +1,76 @@
+{
+  "preamble": {
+    "query_kind": "Binary",
+    "callstack_mode": "Compact"
+  }
+}
+{
+  "vulnerability": {
+    "osv": {
+      "id": "GO-0000-0001",
+      "published": "0001-01-01T00:00:00Z",
+      "modified": "0001-01-01T00:00:00Z",
+      "details": "Third-party vulnerability",
+      "affected": [
+        {
+          "package": {
+            "name": "golang.org/vmod",
+            "ecosystem": ""
+          },
+          "database_specific": {
+            "url": ""
+          },
+          "ecosystem_specific": {
+            "imports": [
+              {
+                "goos": [
+                  "amd"
+                ]
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "modules": [
+      {
+        "path": "golang.org/vmod",
+        "found_version": "v0.0.1",
+        "fixed_version": "v0.1.3"
+      }
+    ]
+  }
+}
+{
+  "vulnerability": {
+    "osv": {
+      "id": "GO-0000-0002",
+      "published": "0001-01-01T00:00:00Z",
+      "modified": "0001-01-01T00:00:00Z",
+      "details": "Stdlib vulnerability",
+      "affected": [
+        {
+          "package": {
+            "name": "stdlib",
+            "ecosystem": ""
+          },
+          "database_specific": {
+            "url": ""
+          },
+          "ecosystem_specific": {}
+        }
+      ]
+    },
+    "modules": [
+      {
+        "path": "stdlib",
+        "found_version": "v0.0.1",
+        "packages": [
+          {
+            "path": "net/http"
+          }
+        ]
+      }
+    ]
+  }
+}

internal/govulncheck/testdata/multi_stacks.json

@@ -0,0 +1,83 @@
+{
+  "preamble": {
+    "query_kind": "Source",
+    "callstack_mode": "Compact"
+  }
+}
+{
+  "vulnerability": {
+    "osv": {
+      "id": "GO-0000-0001",
+      "published": "0001-01-01T00:00:00Z",
+      "modified": "0001-01-01T00:00:00Z",
+      "details": "Third-party vulnerability",
+      "affected": [
+        {
+          "package": {
+            "name": "golang.org/vmod",
+            "ecosystem": ""
+          },
+          "database_specific": {
+            "url": ""
+          },
+          "ecosystem_specific": {
+            "imports": [
+              {
+                "goos": [
+                  "amd"
+                ]
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "modules": [
+      {
+        "path": "golang.org/vmod",
+        "found_version": "v0.0.1",
+        "fixed_version": "v0.1.3",
+        "packages": [
+          {
+            "path": "",
+            "callstacks": [
+              {
+                "symbol": "",
+                "summary": "main calls vmod.Vuln"
+              },
+              {
+                "symbol": "",
+                "summary": "main calls vmod.VulnFoo"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "path": "golang.org/vmod1",
+        "found_version": "v0.0.3",
+        "fixed_version": "v0.0.4",
+        "packages": [
+          {
+            "path": "",
+            "callstacks": [
+              {
+                "symbol": "",
+                "summary": "Foo calls vmod1.Vuln"
+              }
+            ]
+          },
+          {
+            "path": "",
+            "callstacks": [
+              {
+                "symbol": "",
+                "summary": "Bar calls vmod1.VulnFoo"
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

internal/govulncheck/testdata/no_vulns.json

@@ -0,0 +1,43 @@
+{
+  "preamble": {
+    "query_kind": "Source",
+    "callstack_mode": "Compact"
+  }
+}
+{
+  "vulnerability": {
+    "osv": {
+      "id": "GO-0000-0001",
+      "published": "0001-01-01T00:00:00Z",
+      "modified": "0001-01-01T00:00:00Z",
+      "details": "Third-party vulnerability",
+      "affected": [
+        {
+          "package": {
+            "name": "golang.org/vmod",
+            "ecosystem": ""
+          },
+          "database_specific": {
+            "url": ""
+          },
+          "ecosystem_specific": {
+            "imports": [
+              {
+                "goos": [
+                  "amd"
+                ]
+              }
+            ]
+          }
+        }
+      ]
+    },
+    "modules": [
+      {
+        "path": "golang.org/vmod",
+        "found_version": "v0.0.1",
+        "fixed_version": "v0.1.3"
+      }
+    ]
+  }
+}