_cli, _format: Print skipped packages even when no vulns are found

CHANGELOG.md

@@ -52,6 +52,12 @@ All versions prior to 0.0.9 are untracked.
   columnar output format
   ([#232](https://github.com/trailofbits/pip-audit/pull/232))
 
+* CLI: `pip-audit` now prints its output more frequently, including when
+  there are no discovered vulnerabilities but packages were skipped.
+  Similarly, "manifest" output formats (JSON, CycloneDX) are now emitted
+  unconditionally
+  ([#240](https://github.com/trailofbits/pip-audit/pull/240))
+
 ### Fixed
 
 * CLI: A regression causing excess output during `pip audit -r`

pip_audit/_cli.py

@@ -321,13 +321,15 @@ def audit() -> None:
         result = {}
         pkg_count = 0
         vuln_count = 0
+        skip_count = 0
         for (spec, vulns) in auditor.audit(source):
             if spec.is_skipped():
                 spec = cast(SkippedDependency, spec)
                 if args.strict:
                     _fatal(f"{spec.name}: {spec.skip_reason}")
                 else:
                     state.update_state(f"Skipping {spec.name}: {spec.skip_reason}")
+                skip_count += 1
             else:
                 spec = cast(ResolvedDependency, spec)
                 state.update_state(f"Auditing {spec.name} ({spec.version})")
@@ -369,8 +371,6 @@ def audit() -> None:
                         fix = SkippedFixVersion(fix.dep, skip_reason)
                 fixes.append(fix)
 
-    # TODO(ww): Refine this: we should always output if our output format is an SBOM
-    # or other manifest format (like the default JSON format).
     if vuln_count > 0:
         summary_msg = (
             f"Found {vuln_count} known "
@@ -390,3 +390,7 @@ def audit() -> None:
             sys.exit(1)
     else:
         print("No known vulnerabilities found", file=sys.stderr)
+        # If our output format is a "manifest" format we always emit it,
+        # even if nothing other than a dependency summary is present.
+        if skip_count > 0 or formatter.is_manifest:
+            print(formatter.format(result, fixes))

pip_audit/_format/columns.py

@@ -40,6 +40,13 @@ def __init__(self, output_desc: bool):
         """
         self.output_desc = output_desc
 
+    @property
+    def is_manifest(self):
+        """
+        See `VulnerabilityFormat.is_manifest`.
+        """
+        return False
+
     def format(
         self,
         result: Dict[service.Dependency, List[service.VulnerabilityResult]],
@@ -66,17 +73,20 @@ def format(
             for vuln in vulns:
                 vuln_data.append(self._format_vuln(dep, vuln, applied_fix))
 
-        vuln_strings, sizes = tabulate(vuln_data)
+        columns_string = str()
 
-        # Create and add a separator.
-        if len(vuln_data) > 0:
-            vuln_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
+        # If it's just a header, don't bother adding it to the output
+        if len(vuln_data) > 1:
+            vuln_strings, sizes = tabulate(vuln_data)
 
-        columns_string = str()
-        for row in vuln_strings:
-            if columns_string:
-                columns_string += "\n"
-            columns_string += row
+            # Create and add a separator.
+            if len(vuln_data) > 0:
+                vuln_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
+
+            for row in vuln_strings:
+                if columns_string:
+                    columns_string += "\n"
+                columns_string += row
 
         # Now display the skipped dependencies
         skip_data: List[List[Any]] = []
@@ -99,7 +109,9 @@ def format(
         skip_strings.insert(1, " ".join(map(lambda x: "-" * x, sizes)))
 
         for row in skip_strings:
-            columns_string += "\n" + row
+            if columns_string:
+                columns_string += "\n"
+            columns_string += row
 
         return columns_string
 

pip_audit/_format/cyclonedx.py

@@ -68,6 +68,13 @@ def __init__(self, inner_format: "CycloneDxFormat.InnerFormat"):
 
         self._inner_format = inner_format
 
+    @property
+    def is_manifest(self):
+        """
+        See `VulnerabilityFormat.is_manifest`.
+        """
+        return True
+
     def format(
         self,
         result: Dict[service.Dependency, List[service.VulnerabilityResult]],

pip_audit/_format/interface.py

@@ -1,7 +1,7 @@
 """
 Interfaces for formatting vulnerability results into a string representation.
 """
-from abc import ABC
+from abc import ABC, abstractmethod
 from typing import Dict, List
 
 import pip_audit._fix as fix
@@ -13,6 +13,20 @@ class VulnerabilityFormat(ABC):
     Represents an abstract string representation for vulnerability results.
     """
 
+    @property
+    @abstractmethod
+    def is_manifest(self) -> bool:  # pragma: no cover
+        """
+        Is this format a "manifest" format, i.e. one that prints a summary
+        of all results?
+
+        Manifest formats are always rendered emitted unconditionally, even
+        if the audit results contain nothing out of the ordinary
+        (no vulnerabilities, skips, or fixes).
+        """
+        raise NotImplementedError
+
+    @abstractmethod
     def format(
         self,
         result: Dict[service.Dependency, List[service.VulnerabilityResult]],

pip_audit/_format/json.py

@@ -26,6 +26,13 @@ def __init__(self, output_desc: bool):
         """
         self.output_desc = output_desc
 
+    @property
+    def is_manifest(self):
+        """
+        See `VulnerabilityFormat.is_manifest`.
+        """
+        return True
+
     def format(
         self,
         result: Dict[service.Dependency, List[service.VulnerabilityResult]],

test/format/conftest.py

@@ -53,6 +53,17 @@
     _SKIPPED_DEP: [],
 }
 
+_TEST_NO_VULN_DATA: Dict[service.Dependency, List[service.VulnerabilityResult]] = {
+    _RESOLVED_DEP_FOO: [],
+    _RESOLVED_DEP_BAR: [],
+}
+
+_TEST_NO_VULN_DATA_SKIPPED_DEP: Dict[service.Dependency, List[service.VulnerabilityResult]] = {
+    _RESOLVED_DEP_FOO: [],
+    _RESOLVED_DEP_BAR: [],
+    _SKIPPED_DEP: [],
+}
+
 _TEST_FIX_DATA: List[fix.FixVersion] = [
     fix.ResolvedFixVersion(dep=_RESOLVED_DEP_FOO, version=Version("1.8")),
     fix.ResolvedFixVersion(dep=_RESOLVED_DEP_BAR, version=Version("0.3")),
@@ -74,6 +85,16 @@ def vuln_data_skipped_dep():
     return _TEST_VULN_DATA_SKIPPED_DEP
 
 
+@pytest.fixture(autouse=True)
+def no_vuln_data():
+    return _TEST_NO_VULN_DATA
+
+
+@pytest.fixture(autouse=True)
+def no_vuln_data_skipped_dep():
+    return _TEST_NO_VULN_DATA_SKIPPED_DEP
+
+
 @pytest.fixture(autouse=True)
 def fix_data():
     return _TEST_FIX_DATA

test/format/test_columns.py

@@ -1,6 +1,14 @@
+import pytest
+
 import pip_audit._format as format
 
 
+@pytest.mark.parametrize("output_desc", [True, False])
+def test_columns_not_manifest(output_desc):
+    fmt = format.ColumnsFormat(output_desc)
+    assert not fmt.is_manifest
+
+
 def test_columns(vuln_data):
     columns_format = format.ColumnsFormat(True)
     expected_columns = """Name Version ID     Fix Versions Description
@@ -32,6 +40,20 @@ def test_columns_skipped_dep(vuln_data_skipped_dep):
     assert columns_format.format(vuln_data_skipped_dep, list()) == expected_columns
 
 
+def test_columns_no_vuln_data(no_vuln_data):
+    columns_format = format.ColumnsFormat(False)
+    expected_columns = str()
+    assert columns_format.format(no_vuln_data, list()) == expected_columns
+
+
+def test_column_no_vuln_data_skipped_dep(no_vuln_data_skipped_dep):
+    columns_format = format.ColumnsFormat(False)
+    expected_columns = """Name Skip Reason
+---- -----------
+bar  skip-reason"""
+    assert columns_format.format(no_vuln_data_skipped_dep, list()) == expected_columns
+
+
 def test_columns_fix(vuln_data, fix_data):
     columns_format = format.ColumnsFormat(False)
     expected_columns = """Name Version ID     Fix Versions Applied Fix

test/format/test_cyclonedx.py

@@ -2,10 +2,19 @@
 import xml.etree.ElementTree as ET
 
 import pretend  # type: ignore
+import pytest
 
 from pip_audit._format import CycloneDxFormat
 
 
+@pytest.mark.parametrize(
+    "inner", [CycloneDxFormat.InnerFormat.Xml, CycloneDxFormat.InnerFormat.Json]
+)
+def test_cyclonedx_manifest(inner):
+    fmt = CycloneDxFormat(inner_format=inner)
+    assert fmt.is_manifest
+
+
 def test_cyclonedx_inner_json(vuln_data):
     formatter = CycloneDxFormat(inner_format=CycloneDxFormat.InnerFormat.Json)
 

test/format/test_json.py

@@ -1,8 +1,17 @@
 import json
 
+import pytest
+
 import pip_audit._format as format
 
 
+@pytest.mark.parametrize("output_desc", [True, False])
+def test_json_manifest(output_desc):
+    fmt = format.JsonFormat(output_desc)
+
+    assert fmt.is_manifest
+
+
 def test_json(vuln_data):
     json_format = format.JsonFormat(True)
     expected_json = {