From 261a8ffdf850387b075d1df54451a8a02bb90999 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 28 Nov 2024 10:32:03 +1000 Subject: [PATCH 1/5] feat: find repo from latest artifact when provided artifact has none Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 91 ++++++++++++++----- .../repo_finder/repo_finder_deps_dev.py | 66 ++++++++------ src/macaron/repo_finder/repo_finder_java.py | 10 +- src/macaron/repo_finder/repo_utils.py | 78 +++++++++++++++- .../repo_finder_remote_calls/repo_finder.py | 15 +++ 5 files changed, 205 insertions(+), 55 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index d9b4df1e5..cf67e369f 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -45,7 +45,7 @@ from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_finder_java import JavaRepoFinder -from macaron.repo_finder.repo_utils import generate_report, prepare_repo +from macaron.repo_finder.repo_utils import check_repo_urls_are_equal, generate_report, prepare_repo from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR, list_remote_references logger: logging.Logger = logging.getLogger(__name__) @@ -133,7 +133,7 @@ def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: ) -def find_source(purl_string: str, input_repo: str | None) -> bool: +def find_source(purl_string: str, input_repo: str | None, latest_version_fallback: bool = True) -> bool: """Perform repo and commit finding for a passed PURL, or commit finding for a passed PURL and repo. Parameters @@ -142,6 +142,8 @@ def find_source(purl_string: str, input_repo: str | None) -> bool: The PURL string of the target. input_repo: str | None The repository path optionally provided by the user. + latest_version_fallback: bool + A flag that determines whether the latest version of the same artifact can be checked as a fallback option. Returns ------- @@ -151,12 +153,14 @@ def find_source(purl_string: str, input_repo: str | None) -> bool: try: purl = PackageURL.from_string(purl_string) except ValueError as error: - logger.error("Could not parse PURL: %s", error) + logger.error("Could not parse PURL: '%s'. Error: %s", purl_string, error) return False if not purl.version: - logger.debug("PURL is missing version.") - return False + purl = DepsDevRepoFinder().get_latest_version(purl) + if not purl.version: + logger.debug("PURL is missing version.") + return False found_repo = input_repo if not input_repo: @@ -165,11 +169,24 @@ def find_source(purl_string: str, input_repo: str | None) -> bool: if not found_repo: logger.error("Could not find repo for PURL: %s", purl) - return False + if not latest_version_fallback: + return False + + # Try to find the latest version repo. + latest_version_purl = get_latest_version(purl) + if latest_version_purl == purl: + logger.error("Latest version PURL is the same as original: %s", purl) + return False + + found_repo = DepsDevRepoFinder().find_repo(latest_version_purl) + if not found_repo: + logger.error("Could not find repo from latest version of PURL: %s >> %s.", latest_version_purl, purl) + return False # Disable other loggers for cleaner output. logging.getLogger("macaron.slsa_analyzer.analyzer").disabled = True + digest = None if defaults.getboolean("repofinder", "find_source_should_clone"): logger.debug("Preparing repo: %s", found_repo) repo_dir = os.path.join(global_config.output_path, GIT_REPOS_DIR) @@ -180,33 +197,41 @@ def find_source(purl_string: str, input_repo: str | None) -> bool: purl=purl, ) - if not git_obj: - # TODO expand this message to cover cases where the obj was not created due to lack of correct tag. - logger.error("Could not resolve repository: %s", found_repo) - return False - - try: - digest = git_obj.get_head().hash - except ValueError: - logger.debug("Could not retrieve commit hash from repository.") - return False + if git_obj: + try: + digest = git_obj.get_head().hash + except ValueError: + logger.debug("Could not retrieve commit hash from repository.") else: # Retrieve the tags. tags = get_tags_via_git_remote(found_repo) - if not tags: + if tags: + matches = match_tags(list(tags.keys()), purl.name, purl.version) + if matches: + matched_tag = matches[0] + digest = tags[matched_tag] + + if not digest: + logger.error("Could not find commit for purl / repository: %s / %s", purl, found_repo) + if not latest_version_fallback: return False - matches = match_tags(list(tags.keys()), purl.name, purl.version) + # Try to use the latest version of the artifact. + latest_version_purl = get_latest_version(purl) + if latest_version_purl == purl: + logger.error("Latest version PURL is the same as original: %s", purl) + return False - if not matches: + latest_repo = DepsDevRepoFinder().find_repo(latest_version_purl) + if not latest_repo: + logger.error("Could not find repo from latest version of PURL: %s >> %s.", latest_version_purl, purl) return False - matched_tag = matches[0] - digest = tags[matched_tag] + if check_repo_urls_are_equal(found_repo, latest_repo): + logger.error("Latest version repo is the same as original: %s", latest_repo) + return False - if not digest: - logger.error("Could not find commit for purl / repository: %s / %s", purl, found_repo) - return False + return find_source(str(purl), latest_repo, False) if not input_repo: logger.info("Found repository for PURL: %s", found_repo) @@ -219,6 +244,24 @@ def find_source(purl_string: str, input_repo: str | None) -> bool: return True +def get_latest_version(purl: PackageURL) -> PackageURL: + """Get the latest version of the passed artifact. + + Parameters + ---------- + purl: PackageURL + The artifact as a PURL. + + Returns + ------- + PackageURL + The latest version of the same artifact. + """ + namespace = purl.namespace + "/" if purl.namespace else "" + no_version_purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") + return DepsDevRepoFinder.get_latest_version(no_version_purl) + + def get_tags_via_git_remote(repo: str) -> dict[str, str] | None: """Retrieve all tags from a given repository using ls-remote. diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 4696caa27..5164137ab 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -36,6 +36,9 @@ class DepsDevType(StrEnum): class DepsDevRepoFinder(BaseRepoFinder): """This class is used to find repositories using Google's Open Source Insights A.K.A. deps.dev.""" + # See https://docs.deps.dev/api/v3alpha/ + BASE_URL = "https://api.deps.dev/v3alpha/purl/" + def find_repo(self, purl: PackageURL) -> str: """ Attempt to retrieve a repository URL that matches the passed artifact. @@ -108,6 +111,37 @@ def get_project_info(project_url: str) -> dict[str, Any] | None: return response_json + @staticmethod + def get_latest_version(purl: PackageURL) -> PackageURL: + """Return a PURL representing the latest version of the passed artifact.""" + original_purl = purl + if purl.version: + namespace = purl.namespace + "/" if purl.namespace else "" + purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") + + url = f"{DepsDevRepoFinder.BASE_URL}{encode(str(purl), safe='')}" + response = send_get_http_raw(url) + + if not response: + return original_purl + + try: + metadata: dict = json.loads(response.text) + except ValueError as error: + logger.debug("Failed to parse response from deps.dev: %s", error) + return original_purl + + versions_keys = ["package", "versions"] if "package" in metadata else ["version"] + versions = json_extract(metadata, versions_keys, list) + if not versions: + return original_purl + latest_version = json_extract(versions[-1], ["versionKey", "version"], str) + if not latest_version: + return original_purl + + namespace = purl.namespace + "/" if purl.namespace else "" + return PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}@{latest_version}") + def _create_urls(self, purl: PackageURL) -> list[str]: """ Create the urls to search for the metadata relating to the passed artifact. @@ -124,37 +158,13 @@ def _create_urls(self, purl: PackageURL) -> list[str]: list[str] The list of created URLs. """ - # See https://docs.deps.dev/api/v3alpha/ - base_url = f"https://api.deps.dev/v3alpha/purl/{encode(str(purl), safe='')}" + if not purl.version: + purl = DepsDevRepoFinder.get_latest_version(purl) - if not base_url: - return [] - - if purl.version: - return [base_url] - - # Find the latest version. - response = send_get_http_raw(base_url, {}) - - if not response: - return [] - - try: - metadata: dict = json.loads(response.text) - except ValueError as error: - logger.debug("Failed to parse response from deps.dev: %s", error) - return [] - - versions_keys = ["package", "versions"] if "package" in metadata else ["version"] - versions = json_extract(metadata, versions_keys, list) - if not versions: - return [] - latest_version = json_extract(versions[-1], ["versionKey", "version"], str) - if not latest_version: + if not purl.version: return [] - logger.debug("Found latest version: %s", latest_version) - return [f"{base_url}%40{latest_version}"] + return [f"{DepsDevRepoFinder.BASE_URL}{encode(str(purl), safe='')}"] def _retrieve_json(self, url: str) -> str: """ diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 77e1705f8..8d106d1ea 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -11,6 +11,7 @@ from macaron.config.defaults import defaults from macaron.parsers.pomparser import parse_pom_string from macaron.repo_finder.repo_finder_base import BaseRepoFinder +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_validator import find_valid_repository_url from macaron.util import send_get_http_raw @@ -51,8 +52,13 @@ def find_repo(self, purl: PackageURL) -> str: if not version: logger.info("Version missing for maven artifact: %s:%s", group, artifact) - # TODO add support for Java artifacts without a version - return "" + purl = DepsDevRepoFinder().get_latest_version(purl) + if not purl.version: + logger.debug("Could not find version for artifact: %s:%s", purl.namespace, purl.name) + return "" + group = purl.namespace or "" + artifact = purl.name + version = purl.version while group and artifact and version and limit > 0: # Create the URLs for retrieving the artifact's POM diff --git a/src/macaron/repo_finder/repo_utils.py b/src/macaron/repo_finder/repo_utils.py index c3dffc8c5..e81c2148f 100644 --- a/src/macaron/repo_finder/repo_utils.py +++ b/src/macaron/repo_finder/repo_utils.py @@ -15,6 +15,7 @@ from macaron.config.global_config import global_config from macaron.errors import CloneError, RepoCheckOutError from macaron.repo_finder.commit_finder import find_commit +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService from macaron.slsa_analyzer.git_url import ( @@ -131,6 +132,7 @@ def prepare_repo( branch_name: str = "", digest: str = "", purl: PackageURL | None = None, + latest_version_fallback: bool = True, ) -> Git | None: """Prepare the target repository for analysis. @@ -154,6 +156,8 @@ def prepare_repo( The hash of the commit that we want to checkout in the branch. purl : PackageURL | None The PURL of the analysis target. + latest_version_fallback: bool + A flag that determines whether the latest version of the same artifact can be checked as a fallback option. Returns ------- @@ -210,7 +214,12 @@ def prepare_repo( found_digest = find_commit(git_obj, purl) if not found_digest: logger.error("Could not map the input purl string to a specific commit in the corresponding repository.") - return None + if not latest_version_fallback: + return None + # If the commit could not be found, check if the latest version of the artifact has a different repository. + git_obj, repo_path, found_digest = check_latest_version(purl, repo_path, target_dir) + if not git_obj: + return None digest = found_digest # Checking out the specific branch or commit. This operation varies depends on the git service that the @@ -278,3 +287,70 @@ def get_git_service(remote_path: str | None) -> BaseGitService: return git_service return NoneGitService() + + +def check_latest_version(purl: PackageURL, repo_path: str, target_dir: str) -> tuple[Git | None, str, str]: + """Check the latest version of an artifact to see if it has a different repository URL. + + Parameters + ---------- + purl : PackageURL | None + The PURL of the analysis target. + repo_path : str + The path to the repository, can be either local or remote. + target_dir : str + The directory where all remote repository will be cloned. + + Returns + ------- + tuple[Git | None, str, str] + A tuple of: the pydriller.Git object of the repository (or None if error), the repository path, the commit. + """ + namespace = purl.namespace + "/" if purl.namespace else "" + no_version_purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") + + latest_version_purl = DepsDevRepoFinder.get_latest_version(no_version_purl) + if latest_version_purl == purl: + return None, "", "" + + latest_repo = DepsDevRepoFinder().find_repo(latest_version_purl) + if not latest_repo: + return None, "", "" + + if check_repo_urls_are_equal(repo_path, latest_repo): + return None, "", "" + + # Try to prepare the new repo. + git_obj = prepare_repo(target_dir, latest_repo, "", "", purl, False) + if not git_obj: + return None, "", "" + + # Try to find the commit in the new repo. + digest = find_commit(git_obj, purl) + if not digest: + return None, "", "" + + return git_obj, latest_repo, digest + + +def check_repo_urls_are_equal(repo_1: str, repo_2: str) -> bool: + """Check if the two passed repo URLs are equal. + + Parameters + ---------- + repo_1: str + The first repository URL as a string. + repo_2: str + The second repository URL as a string. + + Returns + ------- + bool + True if the repository URLs have equal hostnames and paths, otherwise False. + """ + repo_url_1 = urlparse(repo_1) + repo_url_2 = urlparse(repo_2) + if repo_url_1.hostname != repo_url_2.hostname or repo_url_1.path != repo_url_2.path: + return False + + return True diff --git a/tests/integration/cases/repo_finder_remote_calls/repo_finder.py b/tests/integration/cases/repo_finder_remote_calls/repo_finder.py index 12f10cac1..16de2c7d5 100644 --- a/tests/integration/cases/repo_finder_remote_calls/repo_finder.py +++ b/tests/integration/cases/repo_finder_remote_calls/repo_finder.py @@ -12,6 +12,7 @@ from macaron.config.defaults import defaults from macaron.repo_finder import repo_validator from macaron.repo_finder.repo_finder import find_repo +from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.slsa_analyzer.git_url import clean_url logger: logging.Logger = logging.getLogger(__name__) @@ -70,6 +71,20 @@ def test_repo_finder() -> int: if not parsed_url or not repo_validator.resolve_redirects(parsed_url): return os.EX_UNAVAILABLE + # Test Java package whose SCM metadata only points to the repo in the later versions than is provided here. + purl = PackageURL.from_string("pkg:maven/io.vertx/vertx-auth-common@3.8.0") + repo = find_repo(purl) + if repo == "https://github.com/eclipse-vertx/vertx-auth": + return os.EX_UNAVAILABLE + latest_purl = DepsDevRepoFinder().get_latest_version(purl) + repo = find_repo(latest_purl) + if repo != "https://github.com/eclipse-vertx/vertx-auth": + return os.EX_UNAVAILABLE + + # Test Java package that has no version. + if not find_repo(PackageURL.from_string("pkg:maven/io.vertx/vertx-auth-common")): + return os.EX_UNAVAILABLE + return os.EX_OK From 9966b57d3eb2210bd9160a7b95b6598c3bc1493b Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Mon, 2 Dec 2024 11:24:11 +1000 Subject: [PATCH 2/5] chore: minor fix Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 4 ++-- src/macaron/repo_finder/repo_utils.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index cf67e369f..2a2f80652 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -45,7 +45,7 @@ from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_finder_java import JavaRepoFinder -from macaron.repo_finder.repo_utils import check_repo_urls_are_equal, generate_report, prepare_repo +from macaron.repo_finder.repo_utils import check_repo_urls_are_equivalent, generate_report, prepare_repo from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR, list_remote_references logger: logging.Logger = logging.getLogger(__name__) @@ -227,7 +227,7 @@ def find_source(purl_string: str, input_repo: str | None, latest_version_fallbac logger.error("Could not find repo from latest version of PURL: %s >> %s.", latest_version_purl, purl) return False - if check_repo_urls_are_equal(found_repo, latest_repo): + if check_repo_urls_are_equivalent(found_repo, latest_repo): logger.error("Latest version repo is the same as original: %s", latest_repo) return False diff --git a/src/macaron/repo_finder/repo_utils.py b/src/macaron/repo_finder/repo_utils.py index e81c2148f..59b15c599 100644 --- a/src/macaron/repo_finder/repo_utils.py +++ b/src/macaron/repo_finder/repo_utils.py @@ -317,7 +317,7 @@ def check_latest_version(purl: PackageURL, repo_path: str, target_dir: str) -> t if not latest_repo: return None, "", "" - if check_repo_urls_are_equal(repo_path, latest_repo): + if check_repo_urls_are_equivalent(repo_path, latest_repo): return None, "", "" # Try to prepare the new repo. @@ -333,8 +333,8 @@ def check_latest_version(purl: PackageURL, repo_path: str, target_dir: str) -> t return git_obj, latest_repo, digest -def check_repo_urls_are_equal(repo_1: str, repo_2: str) -> bool: - """Check if the two passed repo URLs are equal. +def check_repo_urls_are_equivalent(repo_1: str, repo_2: str) -> bool: + """Check if the two passed repo URLs are equivalent. Parameters ---------- From 5f941adcf508721128e718d24190fa0eabba8943 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 4 Dec 2024 14:07:22 +1000 Subject: [PATCH 3/5] chore: refactor repo discovery paths Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder.py | 315 ++++++++++++++---- .../repo_finder/repo_finder_deps_dev.py | 32 +- src/macaron/repo_finder/repo_utils.py | 187 +---------- src/macaron/slsa_analyzer/analyzer.py | 3 +- 4 files changed, 281 insertions(+), 256 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder.py b/src/macaron/repo_finder/repo_finder.py index 2a2f80652..29b114a11 100644 --- a/src/macaron/repo_finder/repo_finder.py +++ b/src/macaron/repo_finder/repo_finder.py @@ -36,28 +36,48 @@ import os from urllib.parse import ParseResult, urlunparse +from git import InvalidGitRepositoryError from packageurl import PackageURL +from pydriller import Git from macaron.config.defaults import defaults from macaron.config.global_config import global_config +from macaron.errors import CloneError, RepoCheckOutError from macaron.repo_finder import to_domain_from_known_purl_types -from macaron.repo_finder.commit_finder import match_tags +from macaron.repo_finder.commit_finder import find_commit, match_tags from macaron.repo_finder.repo_finder_base import BaseRepoFinder from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.repo_finder.repo_finder_java import JavaRepoFinder -from macaron.repo_finder.repo_utils import check_repo_urls_are_equivalent, generate_report, prepare_repo -from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR, list_remote_references +from macaron.repo_finder.repo_utils import ( + check_repo_urls_are_equivalent, + generate_report, + get_git_service, + get_local_repos_path, +) +from macaron.slsa_analyzer.git_url import ( + GIT_REPOS_DIR, + check_out_repo_target, + get_remote_origin_of_local_repo, + get_remote_vcs_url, + get_repo_dir_name, + is_empty_repo, + is_remote_repo, + list_remote_references, + resolve_local_path, +) logger: logging.Logger = logging.getLogger(__name__) -def find_repo(purl: PackageURL) -> str: +def find_repo(purl: PackageURL, check_latest_version: bool = True) -> str: """Retrieve the repository URL that matches the given PURL. Parameters ---------- purl : PackageURL The parsed PURL to convert to the repository path. + check_latest_version: bool + A flag that determines whether the latest version of the PURL is also checked. Returns ------- @@ -80,7 +100,22 @@ def find_repo(purl: PackageURL) -> str: # Call Repo Finder and return first valid URL logger.debug("Analyzing %s with Repo Finder: %s", purl, type(repo_finder)) - return repo_finder.find_repo(purl) + found_repo = repo_finder.find_repo(purl) + + if found_repo or not check_latest_version: + return found_repo + + # Try to find the latest version repo. + logger.error("Could not find repo for PURL: %s", purl) + latest_version_purl = get_latest_purl_if_different(purl) + if not latest_version_purl: + logger.debug("Could not find newer PURL than provided: %s", purl) + return "" + + found_repo = DepsDevRepoFinder().find_repo(latest_version_purl) + if not found_repo: + logger.debug("Could not find repo from latest version of PURL: %s", latest_version_purl) + return found_repo def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: @@ -113,8 +148,7 @@ def to_repo_path(purl: PackageURL, available_domains: list[str]) -> str | None: domain = to_domain_from_known_purl_types(purl.type) or (purl.type if purl.type in available_domains else None) if not domain: logger.info("The PURL type of %s is not valid as a repository type. Trying to find the repository...", purl) - # Try to find the repository - return find_repo(purl) + return None if not purl.namespace: logger.error("Expecting a non-empty namespace from %s.", purl) @@ -151,59 +185,54 @@ def find_source(purl_string: str, input_repo: str | None, latest_version_fallbac True if the source was found. """ try: - purl = PackageURL.from_string(purl_string) + purl: PackageURL | None = PackageURL.from_string(purl_string) except ValueError as error: logger.error("Could not parse PURL: '%s'. Error: %s", purl_string, error) return False + if not purl: + # Unreachable. + return False + + checked_latest_purl = False if not purl.version: - purl = DepsDevRepoFinder().get_latest_version(purl) - if not purl.version: - logger.debug("PURL is missing version.") + purl = get_latest_purl_if_different(purl) + if not purl or not purl.version: + logger.error("PURL is missing version.") return False + checked_latest_purl = True found_repo = input_repo - if not input_repo: + if not found_repo: logger.debug("Searching for repo of PURL: %s", purl) found_repo = find_repo(purl) if not found_repo: logger.error("Could not find repo for PURL: %s", purl) - if not latest_version_fallback: - return False - - # Try to find the latest version repo. - latest_version_purl = get_latest_version(purl) - if latest_version_purl == purl: - logger.error("Latest version PURL is the same as original: %s", purl) - return False - - found_repo = DepsDevRepoFinder().find_repo(latest_version_purl) - if not found_repo: - logger.error("Could not find repo from latest version of PURL: %s >> %s.", latest_version_purl, purl) - return False + return False # Disable other loggers for cleaner output. logging.getLogger("macaron.slsa_analyzer.analyzer").disabled = True digest = None if defaults.getboolean("repofinder", "find_source_should_clone"): + # Clone the repo to retrieve the tags. logger.debug("Preparing repo: %s", found_repo) repo_dir = os.path.join(global_config.output_path, GIT_REPOS_DIR) logging.getLogger("macaron.slsa_analyzer.git_url").disabled = True - git_obj = prepare_repo( - repo_dir, - found_repo, - purl=purl, - ) + # The prepare_repo function will also check the latest version of the artifact if required. + git_obj = prepare_repo(repo_dir, found_repo, purl=purl, latest_version_fallback=not checked_latest_purl) if git_obj: try: digest = git_obj.get_head().hash except ValueError: logger.debug("Could not retrieve commit hash from repository.") + + if not digest: + return False else: - # Retrieve the tags. + # Retrieve the tags using a remote git operation. tags = get_tags_via_git_remote(found_repo) if tags: matches = match_tags(list(tags.keys()), purl.name, purl.version) @@ -211,27 +240,21 @@ def find_source(purl_string: str, input_repo: str | None, latest_version_fallbac matched_tag = matches[0] digest = tags[matched_tag] - if not digest: - logger.error("Could not find commit for purl / repository: %s / %s", purl, found_repo) - if not latest_version_fallback: - return False + if not digest: + logger.error("Could not find commit for purl / repository: %s / %s", purl, found_repo) + if not latest_version_fallback or checked_latest_purl: + return False - # Try to use the latest version of the artifact. - latest_version_purl = get_latest_version(purl) - if latest_version_purl == purl: - logger.error("Latest version PURL is the same as original: %s", purl) - return False + # When not cloning the latest version must be checked here. + latest_version_purl = get_latest_purl_if_different(purl) + if not latest_version_purl: + return False - latest_repo = DepsDevRepoFinder().find_repo(latest_version_purl) - if not latest_repo: - logger.error("Could not find repo from latest version of PURL: %s >> %s.", latest_version_purl, purl) - return False + latest_repo = get_latest_repo_if_different(latest_version_purl, found_repo) + if not latest_repo: + return False - if check_repo_urls_are_equivalent(found_repo, latest_repo): - logger.error("Latest version repo is the same as original: %s", latest_repo) - return False - - return find_source(str(purl), latest_repo, False) + return find_source(str(purl), latest_repo, False) if not input_repo: logger.info("Found repository for PURL: %s", found_repo) @@ -244,22 +267,66 @@ def find_source(purl_string: str, input_repo: str | None, latest_version_fallbac return True -def get_latest_version(purl: PackageURL) -> PackageURL: - """Get the latest version of the passed artifact. +def get_latest_purl_if_different(purl: PackageURL) -> PackageURL | None: + """Return the latest version of an artifact represented by a PURL, if it is different. Parameters ---------- - purl: PackageURL - The artifact as a PURL. + purl : PackageURL | None + The PURL of the analysis target. Returns ------- - PackageURL - The latest version of the same artifact. + PackageURL | None + The latest PURL, or None if they are the same or an error occurs. """ - namespace = purl.namespace + "/" if purl.namespace else "" - no_version_purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") - return DepsDevRepoFinder.get_latest_version(no_version_purl) + if purl.version: + namespace = purl.namespace + "/" if purl.namespace else "" + no_version_purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") + else: + no_version_purl = purl + + latest_version_purl = DepsDevRepoFinder.get_latest_version(no_version_purl) + if not latest_version_purl: + logger.error("Latest version PURL could not be found.") + return None + + if latest_version_purl == purl: + logger.error("Latest version PURL is the same as the current.") + return None + + logger.debug("Found new version of PURL: %s", latest_version_purl) + return latest_version_purl + + +def get_latest_repo_if_different(latest_version_purl: PackageURL, original_repo: str) -> str: + """Return the repository of the passed PURL if it is different to the passed repository. + + Parameters + ---------- + latest_version_purl: PackageURL + The PURL to use. + original_repo: str + The repository to compare against. + + Returns + ------- + str + The latest repository, or an empty string if not found. + """ + latest_repo = find_repo(latest_version_purl, False) + if not latest_repo: + logger.error("Could not find repository from latest PURL: %s", latest_version_purl) + return "" + + if check_repo_urls_are_equivalent(original_repo, latest_repo): + logger.error( + "Repository from latest PURL is equivalent to original repository: %s ~= %s", latest_repo, original_repo + ) + return "" + + logger.debug("Found new repository from latest PURL: %s", latest_repo) + return latest_repo def get_tags_via_git_remote(repo: str) -> dict[str, str] | None: @@ -303,3 +370,135 @@ def get_tags_via_git_remote(repo: str) -> dict[str, str] | None: logger.debug("Found %s tags via ls-remote of %s", len(tags), repo) return tags + + +def prepare_repo( + target_dir: str, + repo_path: str, + branch_name: str = "", + digest: str = "", + purl: PackageURL | None = None, + latest_version_fallback: bool = True, +) -> Git | None: + """Prepare the target repository for analysis. + + If ``repo_path`` is a remote path, the target repo is cloned to ``{target_dir}/{unique_path}``. + The ``unique_path`` of a repository will depend on its remote url. + For example, if given the ``repo_path`` https://github.com/org/name.git, it will + be cloned to ``{target_dir}/github.com/org/name``. + + If ``repo_path`` is a local path, this method will check if ``repo_path`` resolves to a directory inside + ``local_repos_path`` and to a valid git repository. + + Parameters + ---------- + target_dir : str + The directory where all remote repository will be cloned. + repo_path : str + The path to the repository, can be either local or remote. + branch_name : str + The name of the branch we want to checkout. + digest : str + The hash of the commit that we want to checkout in the branch. + purl : PackageURL | None + The PURL of the analysis target. + latest_version_fallback: bool + A flag that determines whether the latest version of the same artifact can be checked as a fallback option. + + Returns + ------- + Git | None + The pydriller.Git object of the repository or None if error. + """ + # TODO: separate the logic for handling remote and local repos instead of putting them into this method. + logger.info( + "Preparing the repository for the analysis (path=%s, branch=%s, digest=%s)", + repo_path, + branch_name, + digest, + ) + + resolved_local_path = "" + is_remote = is_remote_repo(repo_path) + + if is_remote: + logger.info("The path to repo %s is a remote path.", repo_path) + resolved_remote_path = get_remote_vcs_url(repo_path) + if not resolved_remote_path: + logger.error("The provided path to repo %s is not a valid remote path.", repo_path) + return None + + git_service = get_git_service(resolved_remote_path) + repo_unique_path = get_repo_dir_name(resolved_remote_path) + resolved_local_path = os.path.join(target_dir, repo_unique_path) + logger.info("Cloning the repository.") + try: + git_service.clone_repo(resolved_local_path, resolved_remote_path) + except CloneError as error: + logger.error("Cannot clone %s: %s", resolved_remote_path, str(error)) + return None + else: + logger.info("Checking if the path to repo %s is a local path.", repo_path) + resolved_local_path = resolve_local_path(get_local_repos_path(), repo_path) + + if resolved_local_path: + try: + git_obj = Git(resolved_local_path) + except InvalidGitRepositoryError: + logger.error("No git repo exists at %s.", resolved_local_path) + return None + else: + logger.error("Error happened while preparing the repo.") + return None + + if is_empty_repo(git_obj): + logger.error("The target repository does not have any commit.") + return None + + # Find the digest and branch if a version has been specified + if not digest and purl and purl.version: + found_digest = find_commit(git_obj, purl) + if not found_digest: + logger.error("Could not map the input purl string to a specific commit in the corresponding repository.") + if not latest_version_fallback: + return None + # If the commit could not be found, check if the latest version of the artifact has a different repository. + latest_purl = get_latest_purl_if_different(purl) + if not latest_purl: + return None + latest_repo = get_latest_repo_if_different(latest_purl, repo_path) + if not latest_repo: + return None + return prepare_repo(latest_repo, latest_repo, target_dir, latest_version_fallback=False) + + digest = found_digest + + # Checking out the specific branch or commit. This operation varies depends on the git service that the + # repository uses. + if not is_remote: + # If the repo path provided by the user is a local path, we need to get the actual origin remote URL of + # the repo to decide on the suitable git service. + origin_remote_url = get_remote_origin_of_local_repo(git_obj) + if is_remote_repo(origin_remote_url): + # The local repo's origin remote url is a remote URL (e.g https://host.com/a/b): In this case, we obtain + # the corresponding git service using ``self.get_git_service``. + git_service = get_git_service(origin_remote_url) + else: + # The local repo's origin remote url is a local path (e.g /path/to/local/...). This happens when the + # target repository is a clone from another local repo or is a clone from a git archive - + # https://git-scm.com/docs/git-archive: In this case, we fall-back to the generic function + # ``git_url.check_out_repo_target``. + if not check_out_repo_target(git_obj, branch_name, digest, not is_remote): + logger.error("Cannot checkout the specific branch or commit of the target repo.") + return None + + return git_obj + + try: + git_service.check_out_repo(git_obj, branch_name, digest, not is_remote) + except RepoCheckOutError as error: + logger.error("Failed to check out repository at %s", resolved_local_path) + logger.error(error) + return None + + return git_obj diff --git a/src/macaron/repo_finder/repo_finder_deps_dev.py b/src/macaron/repo_finder/repo_finder_deps_dev.py index 5164137ab..d66aaaebf 100644 --- a/src/macaron/repo_finder/repo_finder_deps_dev.py +++ b/src/macaron/repo_finder/repo_finder_deps_dev.py @@ -112,9 +112,19 @@ def get_project_info(project_url: str) -> dict[str, Any] | None: return response_json @staticmethod - def get_latest_version(purl: PackageURL) -> PackageURL: - """Return a PURL representing the latest version of the passed artifact.""" - original_purl = purl + def get_latest_version(purl: PackageURL) -> PackageURL | None: + """Return a PURL representing the latest version of the passed artifact. + + Parameters + ---------- + purl : PackageURL + The current PURL. + + Returns + ------- + PackageURL | None + The latest version of the PURL, or None if it could not be found. + """ if purl.version: namespace = purl.namespace + "/" if purl.namespace else "" purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") @@ -123,21 +133,21 @@ def get_latest_version(purl: PackageURL) -> PackageURL: response = send_get_http_raw(url) if not response: - return original_purl + return None try: metadata: dict = json.loads(response.text) except ValueError as error: logger.debug("Failed to parse response from deps.dev: %s", error) - return original_purl + return None versions_keys = ["package", "versions"] if "package" in metadata else ["version"] versions = json_extract(metadata, versions_keys, list) if not versions: - return original_purl + return None latest_version = json_extract(versions[-1], ["versionKey", "version"], str) if not latest_version: - return original_purl + return None namespace = purl.namespace + "/" if purl.namespace else "" return PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}@{latest_version}") @@ -159,10 +169,10 @@ def _create_urls(self, purl: PackageURL) -> list[str]: The list of created URLs. """ if not purl.version: - purl = DepsDevRepoFinder.get_latest_version(purl) - - if not purl.version: - return [] + latest_purl = DepsDevRepoFinder.get_latest_version(purl) + if not latest_purl: + return [] + purl = latest_purl return [f"{DepsDevRepoFinder.BASE_URL}{encode(str(purl), safe='')}"] diff --git a/src/macaron/repo_finder/repo_utils.py b/src/macaron/repo_finder/repo_utils.py index 59b15c599..467776673 100644 --- a/src/macaron/repo_finder/repo_utils.py +++ b/src/macaron/repo_finder/repo_utils.py @@ -8,26 +8,12 @@ import string from urllib.parse import urlparse -from git import InvalidGitRepositoryError from packageurl import PackageURL -from pydriller import Git from macaron.config.global_config import global_config -from macaron.errors import CloneError, RepoCheckOutError -from macaron.repo_finder.commit_finder import find_commit -from macaron.repo_finder.repo_finder_deps_dev import DepsDevRepoFinder from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService from macaron.slsa_analyzer.git_service.base_git_service import NoneGitService -from macaron.slsa_analyzer.git_url import ( - GIT_REPOS_DIR, - check_out_repo_target, - get_remote_origin_of_local_repo, - get_remote_vcs_url, - get_repo_dir_name, - is_empty_repo, - is_remote_repo, - resolve_local_path, -) +from macaron.slsa_analyzer.git_url import GIT_REPOS_DIR logger: logging.Logger = logging.getLogger(__name__) @@ -126,133 +112,6 @@ def create_report(purl: str, commit: str, repo: str) -> str: return json.dumps(data, indent=4) -def prepare_repo( - target_dir: str, - repo_path: str, - branch_name: str = "", - digest: str = "", - purl: PackageURL | None = None, - latest_version_fallback: bool = True, -) -> Git | None: - """Prepare the target repository for analysis. - - If ``repo_path`` is a remote path, the target repo is cloned to ``{target_dir}/{unique_path}``. - The ``unique_path`` of a repository will depend on its remote url. - For example, if given the ``repo_path`` https://github.com/org/name.git, it will - be cloned to ``{target_dir}/github.com/org/name``. - - If ``repo_path`` is a local path, this method will check if ``repo_path`` resolves to a directory inside - ``local_repos_path`` and to a valid git repository. - - Parameters - ---------- - target_dir : str - The directory where all remote repository will be cloned. - repo_path : str - The path to the repository, can be either local or remote. - branch_name : str - The name of the branch we want to checkout. - digest : str - The hash of the commit that we want to checkout in the branch. - purl : PackageURL | None - The PURL of the analysis target. - latest_version_fallback: bool - A flag that determines whether the latest version of the same artifact can be checked as a fallback option. - - Returns - ------- - Git | None - The pydriller.Git object of the repository or None if error. - """ - # TODO: separate the logic for handling remote and local repos instead of putting them into this method. - logger.info( - "Preparing the repository for the analysis (path=%s, branch=%s, digest=%s)", - repo_path, - branch_name, - digest, - ) - - resolved_local_path = "" - is_remote = is_remote_repo(repo_path) - - if is_remote: - logger.info("The path to repo %s is a remote path.", repo_path) - resolved_remote_path = get_remote_vcs_url(repo_path) - if not resolved_remote_path: - logger.error("The provided path to repo %s is not a valid remote path.", repo_path) - return None - - git_service = get_git_service(resolved_remote_path) - repo_unique_path = get_repo_dir_name(resolved_remote_path) - resolved_local_path = os.path.join(target_dir, repo_unique_path) - logger.info("Cloning the repository.") - try: - git_service.clone_repo(resolved_local_path, resolved_remote_path) - except CloneError as error: - logger.error("Cannot clone %s: %s", resolved_remote_path, str(error)) - return None - else: - logger.info("Checking if the path to repo %s is a local path.", repo_path) - resolved_local_path = resolve_local_path(get_local_repos_path(), repo_path) - - if resolved_local_path: - try: - git_obj = Git(resolved_local_path) - except InvalidGitRepositoryError: - logger.error("No git repo exists at %s.", resolved_local_path) - return None - else: - logger.error("Error happened while preparing the repo.") - return None - - if is_empty_repo(git_obj): - logger.error("The target repository does not have any commit.") - return None - - # Find the digest and branch if a version has been specified - if not digest and purl and purl.version: - found_digest = find_commit(git_obj, purl) - if not found_digest: - logger.error("Could not map the input purl string to a specific commit in the corresponding repository.") - if not latest_version_fallback: - return None - # If the commit could not be found, check if the latest version of the artifact has a different repository. - git_obj, repo_path, found_digest = check_latest_version(purl, repo_path, target_dir) - if not git_obj: - return None - digest = found_digest - - # Checking out the specific branch or commit. This operation varies depends on the git service that the - # repository uses. - if not is_remote: - # If the repo path provided by the user is a local path, we need to get the actual origin remote URL of - # the repo to decide on the suitable git service. - origin_remote_url = get_remote_origin_of_local_repo(git_obj) - if is_remote_repo(origin_remote_url): - # The local repo's origin remote url is a remote URL (e.g https://host.com/a/b): In this case, we obtain - # the corresponding git service using ``self.get_git_service``. - git_service = get_git_service(origin_remote_url) - else: - # The local repo's origin remote url is a local path (e.g /path/to/local/...). This happens when the - # target repository is a clone from another local repo or is a clone from a git archive - - # https://git-scm.com/docs/git-archive: In this case, we fall-back to the generic function - # ``git_url.check_out_repo_target``. - if not check_out_repo_target(git_obj, branch_name, digest, not is_remote): - logger.error("Cannot checkout the specific branch or commit of the target repo.") - return None - - return git_obj - - try: - git_service.check_out_repo(git_obj, branch_name, digest, not is_remote) - except RepoCheckOutError as error: - logger.error("Failed to check out repository at %s", resolved_local_path) - logger.error(error) - return None - - return git_obj - - def get_local_repos_path() -> str: """Get the local repos path from global config or use default. @@ -289,50 +148,6 @@ def get_git_service(remote_path: str | None) -> BaseGitService: return NoneGitService() -def check_latest_version(purl: PackageURL, repo_path: str, target_dir: str) -> tuple[Git | None, str, str]: - """Check the latest version of an artifact to see if it has a different repository URL. - - Parameters - ---------- - purl : PackageURL | None - The PURL of the analysis target. - repo_path : str - The path to the repository, can be either local or remote. - target_dir : str - The directory where all remote repository will be cloned. - - Returns - ------- - tuple[Git | None, str, str] - A tuple of: the pydriller.Git object of the repository (or None if error), the repository path, the commit. - """ - namespace = purl.namespace + "/" if purl.namespace else "" - no_version_purl = PackageURL.from_string(f"pkg:{purl.type}/{namespace}{purl.name}") - - latest_version_purl = DepsDevRepoFinder.get_latest_version(no_version_purl) - if latest_version_purl == purl: - return None, "", "" - - latest_repo = DepsDevRepoFinder().find_repo(latest_version_purl) - if not latest_repo: - return None, "", "" - - if check_repo_urls_are_equivalent(repo_path, latest_repo): - return None, "", "" - - # Try to prepare the new repo. - git_obj = prepare_repo(target_dir, latest_repo, "", "", purl, False) - if not git_obj: - return None, "", "" - - # Try to find the commit in the new repo. - digest = find_commit(git_obj, purl) - if not digest: - return None, "", "" - - return git_obj, latest_repo, digest - - def check_repo_urls_are_equivalent(repo_1: str, repo_2: str) -> bool: """Check if the two passed repo URLs are equivalent. diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index a5fd67f22..e95c29a5a 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -43,7 +43,8 @@ extract_repo_and_commit_from_provenance, ) from macaron.repo_finder.provenance_finder import ProvenanceFinder, find_provenance_from_ci -from macaron.repo_finder.repo_utils import get_git_service, prepare_repo +from macaron.repo_finder.repo_finder import prepare_repo +from macaron.repo_finder.repo_utils import get_git_service from macaron.repo_verifier.repo_verifier import verify_repo from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext From b2a997f29d20b598ede41b2b758a6e8dd5928aec Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Wed, 4 Dec 2024 14:55:10 +1000 Subject: [PATCH 4/5] chore: minor fix Signed-off-by: Ben Selwyn-Smith --- src/macaron/repo_finder/repo_finder_java.py | 10 +++++----- .../cases/repo_finder_remote_calls/repo_finder.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/macaron/repo_finder/repo_finder_java.py b/src/macaron/repo_finder/repo_finder_java.py index 8d106d1ea..e6f349d3b 100644 --- a/src/macaron/repo_finder/repo_finder_java.py +++ b/src/macaron/repo_finder/repo_finder_java.py @@ -52,13 +52,13 @@ def find_repo(self, purl: PackageURL) -> str: if not version: logger.info("Version missing for maven artifact: %s:%s", group, artifact) - purl = DepsDevRepoFinder().get_latest_version(purl) - if not purl.version: + latest_purl = DepsDevRepoFinder().get_latest_version(purl) + if not latest_purl or not latest_purl.version: logger.debug("Could not find version for artifact: %s:%s", purl.namespace, purl.name) return "" - group = purl.namespace or "" - artifact = purl.name - version = purl.version + group = latest_purl.namespace or "" + artifact = latest_purl.name + version = latest_purl.version while group and artifact and version and limit > 0: # Create the URLs for retrieving the artifact's POM diff --git a/tests/integration/cases/repo_finder_remote_calls/repo_finder.py b/tests/integration/cases/repo_finder_remote_calls/repo_finder.py index 16de2c7d5..f529cb771 100644 --- a/tests/integration/cases/repo_finder_remote_calls/repo_finder.py +++ b/tests/integration/cases/repo_finder_remote_calls/repo_finder.py @@ -71,12 +71,13 @@ def test_repo_finder() -> int: if not parsed_url or not repo_validator.resolve_redirects(parsed_url): return os.EX_UNAVAILABLE - # Test Java package whose SCM metadata only points to the repo in the later versions than is provided here. + # Test Java package whose SCM metadata only points to the repo in later versions than is provided here. purl = PackageURL.from_string("pkg:maven/io.vertx/vertx-auth-common@3.8.0") repo = find_repo(purl) if repo == "https://github.com/eclipse-vertx/vertx-auth": return os.EX_UNAVAILABLE latest_purl = DepsDevRepoFinder().get_latest_version(purl) + assert latest_purl repo = find_repo(latest_purl) if repo != "https://github.com/eclipse-vertx/vertx-auth": return os.EX_UNAVAILABLE From 608cfe13f14a8fdecacd85b5a370a10127dc894b Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 19 Dec 2024 10:23:13 +1000 Subject: [PATCH 5/5] chore: add comparison test case for source finder vs analyzer Signed-off-by: Ben Selwyn-Smith --- .../latest_repo_comparison/check_output.sh | 6 ++++ .../cases/latest_repo_comparison/test.yaml | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100755 tests/integration/cases/latest_repo_comparison/check_output.sh create mode 100644 tests/integration/cases/latest_repo_comparison/test.yaml diff --git a/tests/integration/cases/latest_repo_comparison/check_output.sh b/tests/integration/cases/latest_repo_comparison/check_output.sh new file mode 100755 index 000000000..c8e9cbf2e --- /dev/null +++ b/tests/integration/cases/latest_repo_comparison/check_output.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +[[ "$(jq -r '.commit' output/reports/maven/io_avaje/avaje-prisms/avaje-prisms.source.json)" = "1f6f953df0b58f0c35b5e136f62f63ba7a22bc03" ]] && +[[ "$(jq -r '.repo' output/reports/maven/io_avaje/avaje-prisms/avaje-prisms.source.json)" = "https://github.com/avaje/avaje-prisms" ]] diff --git a/tests/integration/cases/latest_repo_comparison/test.yaml b/tests/integration/cases/latest_repo_comparison/test.yaml new file mode 100644 index 000000000..3731b88c8 --- /dev/null +++ b/tests/integration/cases/latest_repo_comparison/test.yaml @@ -0,0 +1,36 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Check that the find-source and analyze commands behave the same for a given artifact. + +tags: +- macaron-python-package +- macaron-docker-image + +steps: +- name: Run macaron find source + kind: find-source + options: + command_args: + - -purl + - pkg:maven/io.avaje/avaje-prisms@1.1 +- name: Check that the repository was not cloned + kind: shell + options: + cmd: ls output/git_repos/github.com/avaje/avaje-prisms/ + expect_fail: true +- name: Check the report contents + kind: shell + options: + cmd: ./check_output.sh +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/io.avaje/avaje-prisms@1.1 +- name: Check that correct repository was cloned + kind: shell + options: + cmd: ls output/git_repos/github.com/avaje/avaje-prisms/