feat: add support for externalReferneces for Components and assoc…

…iated enhancements to parsers to obtain information where possible/known

Signed-off-by: Paul Horton <[email protected]>

cyclonedx/model/__init__.py

@@ -17,6 +17,7 @@
 
 import hashlib
 from enum import Enum
+from typing import List, Union
 
 """
 Uniform set of models to represent objects within a CycloneDX software bill-of-materials.
@@ -69,6 +70,36 @@ class HashType:
     _algorithm: HashAlgorithm
     _value: str
 
+    @staticmethod
+    def from_composite_str(composite_hash: str):
+        """
+        Attempts to convert a string which includes both the Hash Algorithm and Hash Value and represent using our
+        internal model classes.
+
+        Args:
+             composite_hash:
+                Composite Hash string of the format `HASH_ALGORITHM`:`HASH_VALUE`.
+                Example: `sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b`.
+
+        Returns:
+            An instance of `HashType` when possible, else `None`.
+        """
+        algorithm: HashAlgorithm = None
+        parts = composite_hash.split(':')
+
+        algorithm_prefix = parts[0].lower()
+        if algorithm_prefix == 'md5':
+            algorithm = HashAlgorithm.MD5
+        elif algorithm_prefix[0:3] == 'sha':
+            algorithm = getattr(HashAlgorithm, 'SHA_{}'.format(algorithm_prefix[3:]))
+        elif algorithm_prefix[0:6] == 'blake2':
+            algorithm = getattr(HashAlgorithm, 'BLAKE2b_{}'.format(algorithm_prefix[6:]))
+
+        return HashType(
+            algorithm=algorithm,
+            hash_value=parts[1].lower()
+        )
+
     def __init__(self, algorithm: HashAlgorithm, hash_value: str):
         self._algorithm = algorithm
         self._value = hash_value
@@ -78,3 +109,95 @@ def get_algorithm(self) -> HashAlgorithm:
 
     def get_hash_value(self) -> str:
         return self._value
+
+
+class ExternalReferenceType(Enum):
+    """
+    Enum object that defines the permissible 'types' for an External Reference according to the CycloneDX schema.
+
+    .. note::
+        See the CycloneDX Schema definition: https://cyclonedx.org/docs/1.3/#type_externalReferenceType
+    """
+    ADVISORIES = 'advisories'
+    BOM = 'bom'
+    BUILD_META = 'build-meta'
+    BUILD_SYSTEM = 'build-system'
+    CHAT = 'chat'
+    DISTRIBUTION = 'distribution'
+    DOCUMENTATION = 'documentation'
+    ISSUE_TRACKER = 'issue-tracker'
+    LICENSE = 'license'
+    MAILING_LIST = 'mailing-list'
+    OTHER = 'other'
+    SOCIAL = 'social'
+    SCM = 'vcs'
+    SUPPORT = 'support'
+    VCS = 'vcs'
+    WEBSITE = 'website'
+
+
+class ExternalReference:
+    """
+    This is out internal representation of an ExternalReference complex type that can be used in multiple places within
+    a CycloneDX BOM document.
+
+    .. note::
+        See the CycloneDX Schema definition: https://cyclonedx.org/docs/1.3/#type_externalReference
+    """
+    _reference_type: ExternalReferenceType
+    _url: str
+    _comment: str
+    _hashes: List[HashType] = []
+
+    def __init__(self, reference_type: ExternalReferenceType, url: str, comment: str = None,
+                 hashes: List[HashType] = []):
+        self._reference_type = reference_type
+        self._url = url
+        self._comment = comment
+        self._hashes = hashes
+
+    def add_hash(self, our_hash: HashType):
+        """
+        Adds a hash that pins/identifies this External Reference.
+
+        Args:
+            our_hash:
+                `HashType` instance
+        """
+        self._hashes.append(our_hash)
+
+    def get_comment(self) -> Union[str, None]:
+        """
+        Get the comment for this External Reference.
+
+        Returns:
+            Any comment as a `str` else `None`.
+        """
+        return self._comment
+
+    def get_hashes(self) -> List[HashType]:
+        """
+        List of cryptographic hashes that identify this External Reference.
+
+        Returns:
+            `List` of `HashType` objects where there are any hashes, else an empty `List`.
+        """
+        return self._hashes
+
+    def get_reference_type(self) -> ExternalReferenceType:
+        """
+        Get the type of this External Reference.
+
+        Returns:
+            `ExternalReferenceType` that represents the type of this External Reference.
+        """
+        return self._reference_type
+
+    def get_url(self) -> str:
+        """
+        Get the URL/URI for this External Reference.
+
+        Returns:
+            URI as a `str`.
+        """
+        return self._url

cyclonedx/model/component.py

@@ -22,7 +22,7 @@
 from packageurl import PackageURL
 from typing import List
 
-from . import HashAlgorithm, HashType, sha1sum
+from . import ExternalReference, HashAlgorithm, HashType, sha1sum
 from .vulnerability import Vulnerability
 
 
@@ -62,6 +62,7 @@ class Component:
 
     _hashes: List[HashType] = []
     _vulnerabilites: List[Vulnerability] = []
+    _external_references: List[ExternalReference] = []
 
     @staticmethod
     def for_file(absolute_file_path: str, path_for_bom: str = None):
@@ -92,15 +93,28 @@ def for_file(absolute_file_path: str, path_for_bom: str = None):
             package_url_type='generic'
         )
 
-    def __init__(self, name: str, version: str, qualifiers: str = None, hashes: List[HashType] = [],
+    def __init__(self, name: str, version: str, qualifiers: str = None, hashes: List[HashType] = None,
                  component_type: ComponentType = ComponentType.LIBRARY, package_url_type: str = 'pypi'):
         self._name = name
         self._version = version
         self._type = component_type
         self._qualifiers = qualifiers
-        self._hashes = hashes
-        self._vulnerabilites = []
+        self._hashes.clear()
+        if hashes:
+            self._hashes = hashes
+        self._vulnerabilites.clear()
         self._package_url_type = package_url_type
+        self._external_references.clear()
+
+    def add_external_reference(self, reference: ExternalReference):
+        """
+        Add an `ExternalReference` to this `Component`.
+
+        Args:
+            reference:
+                `ExternalReference` instance to add.
+        """
+        self._external_references.append(reference)
 
     def add_hash(self, hash: HashType):
         """
@@ -143,6 +157,15 @@ def get_description(self) -> str:
         """
         return self._description
 
+    def get_external_references(self) -> List[ExternalReference]:
+        """
+        List of external references for this Component.
+
+        Returns:
+            `List` of `ExternalReference` objects where there are any, else an empty `List`.
+        """
+        return self._external_references
+
     def get_hashes(self) -> List[HashType]:
         """
         List of cryptographic hashes that identify this Component.
@@ -182,6 +205,9 @@ def get_purl(self) -> str:
             base_purl = '{}?{}'.format(base_purl, self._qualifiers)
         return base_purl
 
+    def get_pypi_url(self) -> str:
+        return f'https://pypi.org/project/{self.get_name()}/{self.get_version()}'
+
     def get_type(self) -> ComponentType:
         """
         Get the type of this Component.

cyclonedx/output/json.py

@@ -53,7 +53,7 @@ def _get_component_as_dict(self, component: Component) -> dict:
             "purl": component.get_purl()
         }
 
-        if len(component.get_hashes()) > 0:
+        if component.get_hashes():
             hashes = []
             for component_hash in component.get_hashes():
                 hashes.append({
@@ -62,9 +62,31 @@ def _get_component_as_dict(self, component: Component) -> dict:
                 })
             c['hashes'] = hashes
 
-        if self.component_supports_author() and component.get_author() is not None:
+        if self.component_supports_author() and component.get_author():
             c['author'] = component.get_author()
 
+        if self.component_supports_external_references() and component.get_external_references():
+            c['externalReferences'] = []
+            for ext_ref in component.get_external_references():
+                ref = {
+                    "type": ext_ref.get_reference_type().value,
+                    "url": ext_ref.get_url()
+                }
+
+                if ext_ref.get_comment():
+                    ref['comment'] = ext_ref.get_comment()
+
+                if ext_ref.get_hashes():
+                    ref_hashes = []
+                    for ref_hash in ext_ref.get_hashes():
+                        ref_hashes.append({
+                            "alg": ref_hash.get_algorithm().value,
+                            "content": ref_hash.get_hash_value()
+                        })
+                    ref['hashes'] = ref_hashes
+
+                c['externalReferences'].append(ref)
+
         return c
 
     def _get_metadata_as_dict(self) -> dict:

cyclonedx/output/schema.py

@@ -34,6 +34,9 @@ def component_supports_author(self) -> bool:
     def component_supports_bom_ref(self) -> bool:
         return True
 
+    def component_supports_external_references(self) -> bool:
+        return True
+
     def get_schema_version(self) -> str:
         pass
 
@@ -79,5 +82,8 @@ def component_supports_author(self) -> bool:
     def component_supports_bom_ref(self) -> bool:
         return False
 
+    def component_supports_external_references(self) -> bool:
+        return False
+
     def get_schema_version(self) -> str:
         return '1.0'

cyclonedx/output/xml.py

@@ -17,10 +17,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (c) OWASP Foundation. All Rights Reserved.
 
+from typing import List
 from xml.etree import ElementTree
 
 from . import BaseOutput
 from .schema import BaseSchemaVersion, SchemaVersion1Dot0, SchemaVersion1Dot1, SchemaVersion1Dot2, SchemaVersion1Dot3
+from ..model import HashType
 from ..model.component import Component
 from ..model.vulnerability import Vulnerability, VulnerabilityRating
 
@@ -89,16 +91,32 @@ def _get_component_as_xml_element(self, component: Component) -> ElementTree.Ele
         # version
         ElementTree.SubElement(component_element, 'version').text = component.get_version()
 
+        # hashes
+        if len(component.get_hashes()) > 0:
+            Xml._add_hashes_to_element(hashes=component.get_hashes(), element=component_element)
+            # hashes_e = ElementTree.SubElement(component_element, 'hashes')
+            # for hash in component.get_hashes():
+            #     ElementTree.SubElement(
+            #         hashes_e, 'hash', {'alg': hash.get_algorithm().value}
+            #     ).text = hash.get_hash_value()
+
         # purl
         ElementTree.SubElement(component_element, 'purl').text = component.get_purl()
 
-        # hashes
-        if len(component.get_hashes()) > 0:
-            hashes_e = ElementTree.SubElement(component_element, 'hashes')
-            for hash in component.get_hashes():
-                ElementTree.SubElement(
-                    hashes_e, 'hash', {'alg': hash.get_algorithm().value}
-                ).text = hash.get_hash_value()
+        # externalReferences
+        if self.component_supports_external_references() and len(component.get_external_references()) > 0:
+            external_references_e = ElementTree.SubElement(component_element, 'externalReferences')
+            for ext_ref in component.get_external_references():
+                external_reference_e = ElementTree.SubElement(
+                    external_references_e, 'reference', {'type': ext_ref.get_reference_type().value}
+                )
+                ElementTree.SubElement(external_reference_e, 'url').text = ext_ref.get_url()
+
+                if ext_ref.get_comment():
+                    ElementTree.SubElement(external_reference_e, 'comment').text = ext_ref.get_comment()
+
+                if len(ext_ref.get_hashes()) > 0:
+                    Xml._add_hashes_to_element(hashes=ext_ref.get_hashes(), element=external_reference_e)
 
         return component_element
 
@@ -194,6 +212,14 @@ def _add_metadata(self, bom: ElementTree.Element) -> ElementTree.Element:
 
         return bom
 
+    @staticmethod
+    def _add_hashes_to_element(hashes: List[HashType], element: ElementTree.Element):
+        hashes_e = ElementTree.SubElement(element, 'hashes')
+        for h in hashes:
+            ElementTree.SubElement(
+                hashes_e, 'hash', {'alg': h.get_algorithm().value}
+            ).text = h.get_hash_value()
+
 
 class XmlV1Dot0(Xml, SchemaVersion1Dot0):
 

cyclonedx/parser/__init__.py

@@ -18,6 +18,50 @@
 Set of classes and methods which allow for quick creation of a Bom instance from your environment or Python project.
 
 Use a Parser instead of programmatically creating a Bom as a developer.
+
+Different parsers support population of different information about your dependencies due to how information is
+obtained and limitations of what information is available to each Parser. The table below explains coverage as to what
+information is obtained by each set of Parsers. It does NOT guarantee the information is output in the resulting
+CycloneDX BOM document.
+
+| Data Path | Environment | Pipenv | Poetry | Requirements |
+| ----------- | ----------- | ----------- | ----------- | ----------- |
+| `component.supplier` | N (if in package METADATA) | N/A | | |
+| `component.author` | Y (if in package METADATA) | N/A | | |
+| `component.publisher` | N (if in package METADATA) | N/A | | |
+| `component.group` | - | - | - | - |
+| `component.name` | Y | Y | Y | Y |
+| `component.version` | Y | Y | Y | Y |
+| `component.description` | N | N/A | N | N/A |
+| `component.scope` | N | N/A | N | N/A |
+| `component.hashes` | N/A | Y - see below (1) | Y - see below (1) | N/A |
+| `component.licenses` | Y (if in package METADATA) | N/A | N/A | N/A |
+| `component.copyright` | N (if in package METADATA) | N/A | N/A | N/A |
+| `component.cpe` | _Deprecated_ | _Deprecated_ | _Deprecated_ | _Deprecated_ |
+| `component.purl` | Y | Y | Y | Y |
+| `component.swid` | N/A | N/A | N/A | N/A |
+| `component.modified` | _Deprecated_ | _Deprecated_ | _Deprecated_ | _Deprecated_ |
+| `component.pedigree` | N/A | N/A | N/A | N/A |
+| `component.externalReferences` | N/A | Y - see below (1) | Y - see below (1) | N/A |
+| `component.properties` | N/A | N/A | N/A | N/A |
+| `component.components` | N/A | N/A | N/A | N/A |
+| `component.evidence` | N/A | N/A | N/A | N/A |
+
+**Legend:**
+
+* `Y`: YES with any caveat states.
+* `N`: Not currently supported, but support believed to be possible.
+* `N/A`: Not supported and not deemed possible (i.e. the Parser would never be able to reliably determine this info).
+* `-`: Deemed not applicable to the Python ecosystem.
+
+**Notes:**
+
+1. Python packages are regularly available as both `.whl` and `.tar.gz` packages. This means for that for a given
+    package and version multiple artefacts are possible - which would mean multiple hashes are possible. CycloneDX
+    supports only a single set of hashes identifying a single artefact at `component.hashes`. To cater for this
+    situation in Python, we add the hashes to `component.externalReferences`, as we cannot determine which package was
+    actually obtained and installed to meet a given dependency.
+
 """
 
 from abc import ABC