diff --git a/.gitignore b/.gitignore index d4cb9d1..bc385a3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ dist/ .hypothesis/ .coverage prof/ +htmlcov/ +.pytest_cache/ \ No newline at end of file diff --git a/docs/source/examples/complex_dependencies.py b/docs/source/examples/complex_dependencies.py index 371768c..406631b 100644 --- a/docs/source/examples/complex_dependencies.py +++ b/docs/source/examples/complex_dependencies.py @@ -1,7 +1,6 @@ from jaypore_ci import jci with jci.Pipeline() as p: - with p.stage("build"): p.job("DockDev", f"docker build --target DevEnv -t {p.repo.sha}_dev .") diff --git a/docs/source/examples/custom_services.py b/docs/source/examples/custom_services.py index a90ff54..6bdcd3e 100644 --- a/docs/source/examples/custom_services.py +++ b/docs/source/examples/custom_services.py @@ -4,7 +4,6 @@ # If they exit with a Non ZERO code they are marked as FAILED, otherwise # they are assumed to be PASSED with jci.Pipeline() as p: - # Since we define all jobs in this section as `is_service=True`, they will # keep running for as long as the pipeline runs. with p.stage("Services", is_service=True): diff --git a/jaypore_ci/interfaces.py b/jaypore_ci/interfaces.py index 6f7db59..05ea1f9 100644 --- a/jaypore_ci/interfaces.py +++ b/jaypore_ci/interfaces.py @@ -58,32 +58,46 @@ def parse(cls, remote: str) -> "RemoteInfo": ssh://git@gitea.arjoonn.com:arjoonn/jaypore_ci.git ssh+git://git@gitea.arjoonn.com:arjoonn/jaypore_ci.git - git@gitea.arjoonn.com:arjoonn/jaypore_ci.git git@gitea.arjoonn.com:arjoonn/jaypore_ci.git https://gitea.arjoonn.com/midpath/jaypore_ci.git http://gitea.arjoonn.com/midpath/jaypore_ci.git """ - original = remote - if ( - ("ssh://" in remote or "ssh+git://" in remote or "://" not in remote) - and "@" in remote - and remote.endswith(".git") - ): - _, remote = remote.split("@") - netloc, path = remote.split(":") - owner, repo = path.split("/") - return RemoteInfo( - netloc=netloc, - owner=owner, - repo=repo.replace(".git", ""), - original=original, - ) + if cls._is_remote_ssh(remote): + return cls._parse_ssh_remote(remote) + + assert ( + "https://" in remote or "http://" in remote + ) and ".git" in remote, ( + f"Only https, http & ssh remotes are supported. (Remote: {remote})" + ) + url = urlparse(remote) return RemoteInfo( netloc=url.netloc, owner=Path(url.path).parts[1], repo=Path(url.path).parts[2].replace(".git", ""), + original=remote, + ) + + @classmethod + def _is_remote_ssh(cls, remote: str) -> bool: + return ( + ("ssh://" in remote or "ssh+git://" in remote or "://" not in remote) + and "@" in remote + and remote.endswith(".git") + ) + + @classmethod + def _parse_ssh_remote(cls, remote: str) -> "RemoteInfo": + original = remote + _, remote = remote.split("@") + netloc, path = remote.split(":") + owner, repo = path.split("/") + return RemoteInfo( + netloc=netloc, + owner=owner, + repo=repo.replace(".git", ""), original=original, ) @@ -96,21 +110,28 @@ class Repo: def __init__(self, sha: str, branch: str, remote: str, commit_message: str): self.sha: str = sha self.branch: str = branch - self.remote: str = remote + self.remote: RemoteInfo = RemoteInfo.parse(remote) self.commit_message: str = commit_message - def files_changed(self, target: str) -> List[str]: + @classmethod + def from_env(cls) -> "Repo": """ - Returns list of file paths that have changed between current sha and - target. + Creates a :class:`~jaypore_ci.interfaces.Repo` instance + from the environment and git repo on disk. """ raise NotImplementedError() @classmethod - def from_env(cls) -> "Repo": + def _get_remote_url(cls) -> str: """ - Creates a :class:`~jaypore_ci.interfaces.Repo` instance - from the environment and git repo on disk. + Returns remote URL from the repo on disk. + """ + raise NotImplementedError() + + def files_changed(self, target: str) -> List[str]: + """ + Returns list of file paths that have changed between current sha and + target. """ raise NotImplementedError() @@ -159,8 +180,8 @@ class Remote: """ def __init__(self, *, sha, branch): - self.sha = sha self.branch = branch + self.sha = sha def publish(self, report: str, status: str): """ diff --git a/jaypore_ci/jci.py b/jaypore_ci/jci.py index 0f1dd3c..72fa109 100644 --- a/jaypore_ci/jci.py +++ b/jaypore_ci/jci.py @@ -34,6 +34,7 @@ FIN_STATUSES = (Status.FAILED, Status.PASSED, Status.TIMEOUT, Status.SKIPPED) PREFIX = "JAYPORE_" + # Check if we need to upgrade Jaypore CI def ensure_version_is_correct(): """ diff --git a/jaypore_ci/remotes/email.py b/jaypore_ci/remotes/email.py index a2da93b..ddd77ef 100644 --- a/jaypore_ci/remotes/email.py +++ b/jaypore_ci/remotes/email.py @@ -68,9 +68,6 @@ def from_env(cls, *, repo: Repo) -> "Email": """ Creates a remote instance from the environment. """ - remote = urlparse(repo.remote) - owner = Path(remote.path).parts[1] - name = Path(remote.path).parts[2].replace(".git", "") return cls( host=os.environ.get("JAYPORE_EMAIL_HOST", "smtp.gmail.com"), port=int(os.environ.get("JAYPORE_EMAIL_PORT", 465)), @@ -80,7 +77,7 @@ def from_env(cls, *, repo: Repo) -> "Email": email_from=os.environ.get( "JAYPORE_EMAIL_FROM", os.environ["JAYPORE_EMAIL_ADDR"] ), - subject=f"JCI [{owner}/{name}] [{repo.branch} {repo.sha[:8]}]", + subject=f"JCI [{repo.remote.owner}/{repo.remote.repo}] [{repo.branch} {repo.sha[:8]}]", branch=repo.branch, sha=repo.sha, ) diff --git a/jaypore_ci/remotes/gitea.py b/jaypore_ci/remotes/gitea.py index 7ced76b..67bfd49 100644 --- a/jaypore_ci/remotes/gitea.py +++ b/jaypore_ci/remotes/gitea.py @@ -29,11 +29,10 @@ def from_env(cls, *, repo: Repo) -> "Gitea": """ os.environ["JAYPORE_COMMIT_BRANCH"] = repo.branch os.environ["JAYPORE_COMMIT_SHA"] = repo.sha - rem = RemoteInfo.parse(repo.remote) return cls( - root=f"https://{rem.netloc}", - owner=rem.owner, - repo=rem.repo, + root=f"https://{repo.remote.netloc}", + owner=repo.remote.owner, + repo=repo.remote.repo, branch=repo.branch, token=os.environ["JAYPORE_GITEA_TOKEN"], sha=repo.sha, diff --git a/jaypore_ci/remotes/github.py b/jaypore_ci/remotes/github.py index f3f2585..1117688 100644 --- a/jaypore_ci/remotes/github.py +++ b/jaypore_ci/remotes/github.py @@ -34,13 +34,12 @@ def from_env(cls, *, repo: Repo) -> "Github": - Create a new pull request for that branch - Allow posting updates using the gitea token provided """ - rem = RemoteInfo.parse(repo.remote) os.environ["JAYPORE_COMMIT_BRANCH"] = repo.branch os.environ["JAYPORE_COMMIT_SHA"] = repo.sha return cls( - root="https://api.github.com", - owner=rem.owner, - repo=rem.repo, + root=f"https://{repo.remote.netloc}", + owner=repo.remote.owner, + repo=repo.remote.repo, branch=repo.branch, token=os.environ["JAYPORE_GITHUB_TOKEN"], sha=repo.sha, diff --git a/jaypore_ci/repos/git.py b/jaypore_ci/repos/git.py index 5eff40d..f3c46a7 100644 --- a/jaypore_ci/repos/git.py +++ b/jaypore_ci/repos/git.py @@ -1,3 +1,4 @@ +import os import subprocess from typing import List @@ -21,18 +22,8 @@ def from_env(cls) -> "Git": """ Gets repo status from the environment and git repo on disk. """ - remote = ( - subprocess.check_output( - "git remote -v | grep push | head -n1 | grep https | awk '{print $2}'", - shell=True, - ) - .decode() - .strip() - ) - assert "https://" in remote, f"Only https remotes supported: {remote}" - assert ".git" in remote - # NOTE: Later on perhaps we should support non-https remotes as well - # since JCI does not actually do anything with the remote. + remote = cls._get_remote_url() + assert remote, "Obtained remote url for pushing is empty!" branch = ( subprocess.check_output( r"git branch | grep \* | awk '{print $2}'", shell=True @@ -47,3 +38,15 @@ def from_env(cls) -> "Git": .strip() ) return cls(sha=sha, branch=branch, remote=remote, commit_message=message) + + @classmethod + def _get_remote_url(cls) -> str: + if os.environ.get("JAYPORECI_DOCS_EXAMPLES_TEST_MODE", False): + return "https://test-mode.com/test/test-remote.git" + return ( + subprocess.check_output( + "git remote -v | grep push | head -n1 | awk '{print $2}'", shell=True + ) + .decode() + .strip() + ) diff --git a/tests/conftest.py b/tests/conftest.py index 75bf17a..933002d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,9 @@ from jaypore_ci import jci, executors, remotes, reporters, repos +from typing import Callable +from jaypore_ci.interfaces import Repo, Remote, Executor, Reporter + def idfn(x): name = [] @@ -19,10 +22,19 @@ def idfn(x): return str(name) -def factory(*, repo, remote, executor, reporter): +def factory( + *, + repo: Repo, + remote: Remote, + remote_url: str, + executor: Executor, + reporter: Reporter +) -> Callable: "Return a new pipeline every time the builder function is called" - def build(): + repo._get_remote_url = lambda: remote_url + + def build() -> jci.Pipeline: r = repo.from_env() return jci.Pipeline( poll_interval=0, @@ -47,6 +59,7 @@ def set_env_keys(): scope="function", params=list( jci.Pipeline.env_matrix( + executor=[executors.Docker], reporter=[reporters.Text, reporters.Markdown], remote=[ remotes.Mock, @@ -55,8 +68,14 @@ def set_env_keys(): remotes.Gitea, remotes.Github, ], + remote_url=[ + "https://fake-remote.com/fake_owner/fake_repo.git", + "http://fake-remote.com/midpath/jaypore_ci.git", + "ssh://user@fake-remote.com:fake_owner/fake_repo.git", + "ssh+git://user@fake-remote.com:fake_owner/fake_repo.git", + "user@fake-remote.com:fake_owner/fake_repo.git", + ], repo=[repos.Git], - executor=[executors.Docker], ) ), ids=idfn, @@ -66,13 +85,21 @@ def pipeline(request): builder = factory( repo=request.param["repo"], remote=request.param["remote"], + remote_url=request.param["remote_url"], executor=request.param["executor"], reporter=request.param["reporter"], ) - if request.param["remote"] == remotes.Gitea and not Mock.gitea_added: - add_gitea_mocks(builder().remote) - if request.param["remote"] == remotes.Github and not Mock.github_added: - add_github_mocks(builder().remote) + if ( + request.param["remote"] == remotes.Gitea + and not request.param["remote_url"] in Mock.gitea_remotes + ): + add_gitea_mocks(builder().remote, request.param["remote_url"]) + if ( + request.param["remote"] == remotes.Github + and not request.param["remote_url"] in Mock.github_remotes + ): + add_github_mocks(builder().remote, request.param["remote_url"]) + if request.param["remote"] == remotes.Email: with unittest.mock.patch("smtplib.SMTP_SSL", autospec=True): yield builder @@ -87,4 +114,6 @@ def pipeline(request): ) def doc_example_filepath(request): set_env_keys() + os.environ["JAYPORECI_DOCS_EXAMPLES_TEST_MODE"] = "1" yield request.param + os.environ.pop("JAYPORECI_DOCS_EXAMPLES_TEST_MODE") diff --git a/tests/requests_mock.py b/tests/requests_mock.py index 5037463..b4ce0f7 100644 --- a/tests/requests_mock.py +++ b/tests/requests_mock.py @@ -1,9 +1,10 @@ import json +import requests -from typing import NamedTuple +from typing import NamedTuple, Callable, List from collections import defaultdict -import requests +from jaypore_ci.remotes import Gitea, Github class MockResponse(NamedTuple): @@ -11,41 +12,59 @@ class MockResponse(NamedTuple): body: str content_type: str - def json(self): + def json(self) -> str: return json.loads(self.body) @property - def text(self): + def text(self) -> str: return self.body class Mock: registry = defaultdict(list) index = defaultdict(int) - gitea_added = False - github_added = False + gitea_remotes: List[str] = [] + github_remotes: List[str] = [] @classmethod - def get(cls, url, status=200, body="", content_type="text/html"): + def get( + cls, + url: str, + status: int = 200, + body: str = "", + content_type: str = "text/html", + ) -> None: cls.registry["get", url].append( MockResponse(status_code=status, body=body, content_type=content_type) ) @classmethod - def post(cls, url, status=200, body="", content_type="text/html"): + def post( + cls, + url: str, + status: int = 200, + body: str = "", + content_type: str = "text/html", + ) -> None: cls.registry["post", url].append( MockResponse(status_code=status, body=body, content_type=content_type) ) @classmethod - def patch(cls, url, status=200, body="", content_type="text/html"): + def patch( + cls, + url: str, + status: int = 200, + body: str = "", + content_type: str = "text/html", + ) -> None: cls.registry["patch", url].append( MockResponse(status_code=status, body=body, content_type=content_type) ) @classmethod - def handle(cls, method): - def handler(url, **_): + def handle(cls, method: str) -> Callable: + def handler(url: str, **_): options = cls.registry[method, url] index = cls.index[method, url] resp = options[index] @@ -55,7 +74,7 @@ def handler(url, **_): return handler -def add_gitea_mocks(gitea): +def add_gitea_mocks(gitea: Gitea, remote_url: str) -> None: ISSUE_ID = 1 # --- create PR create_pr_url = f"{gitea.api}/repos/{gitea.owner}/{gitea.repo}/pulls" @@ -71,10 +90,10 @@ def add_gitea_mocks(gitea): Mock.patch(f"{gitea.api}/repos/{gitea.owner}/{gitea.repo}/pulls/{ISSUE_ID}") # --- set commit status Mock.post(f"{gitea.api}/repos/{gitea.owner}/{gitea.repo}/statuses/{gitea.sha}") - Mock.gitea_added = True + Mock.gitea_remotes.append(remote_url) -def add_github_mocks(github): +def add_github_mocks(github: Github, remote_url: str) -> None: ISSUE_ID = 1 # --- create PR create_pr_url = f"{github.api}/repos/{github.owner}/{github.repo}/pulls" @@ -95,7 +114,7 @@ def add_github_mocks(github): Mock.patch(f"{github.api}/repos/{github.owner}/{github.repo}/pulls/{ISSUE_ID}") # --- set commit status Mock.post(f"{github.api}/repos/{github.owner}/{github.repo}/statuses/{github.sha}") - Mock.github_added = True + Mock.github_remotes.append(remote_url) requests.get = Mock.handle("get")