From 1a95f476bd8d79c50c698004e58dfad33478f45f Mon Sep 17 00:00:00 2001 From: Thomas Desveaux Date: Mon, 8 Jul 2024 20:24:16 +0200 Subject: [PATCH] gitpoller: implement git auth with user/password --- master/buildbot/changes/gitpoller.py | 23 +++++- .../test/unit/changes/test_gitpoller.py | 72 +++++++++++++++++++ master/buildbot/util/git.py | 22 +++++- .../manual/configuration/changesources.rst | 10 +++ .../gitpoller-credential-auth.feature | 1 + 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 newsfragments/gitpoller-credential-auth.feature diff --git a/master/buildbot/changes/gitpoller.py b/master/buildbot/changes/gitpoller.py index 0b111afd85e5..e5e6c6f2d929 100644 --- a/master/buildbot/changes/gitpoller.py +++ b/master/buildbot/changes/gitpoller.py @@ -30,9 +30,12 @@ from buildbot.util import giturlparse from buildbot.util import private_tempdir from buildbot.util import runprocess +from buildbot.util import unicode2bytes from buildbot.util.git import GitMixin from buildbot.util.git import GitServiceAuth from buildbot.util.git import check_ssh_config +from buildbot.util.git_credential import GitCredentialOptions +from buildbot.util.git_credential import add_user_password_to_credentials from buildbot.util.state import StateMixin from buildbot.util.twisted import async_to_deferred @@ -40,6 +43,8 @@ from typing import Callable from typing import Literal + from buildbot.interfaces import IRenderable + class GitError(Exception): """Raised when git exits with code 128.""" @@ -97,6 +102,8 @@ def checkConfig( # type: ignore[override] sshKnownHosts=None, pollRandomDelayMin=0, pollRandomDelayMax=0, + auth_credentials: tuple[IRenderable | str, IRenderable | str] | None = None, + git_credentials: GitCredentialOptions | None = None, ): if only_tags and (branch or branches): config.error("GitPoller: can't specify only_tags and branch/branches") @@ -156,6 +163,8 @@ def reconfigService( sshKnownHosts=None, pollRandomDelayMin=0, pollRandomDelayMax=0, + auth_credentials: tuple[IRenderable | str, IRenderable | str] | None = None, + git_credentials: GitCredentialOptions | None = None, ): if name is None: name = repourl @@ -186,7 +195,17 @@ def reconfigService( self.lastRev = None self.setupGit() - self._git_auth = GitServiceAuth(self, sshPrivateKey, sshHostKey, sshKnownHosts) + + if auth_credentials is not None: + git_credentials = add_user_password_to_credentials( + auth_credentials, + repourl, + git_credentials, + ) + + self._git_auth = GitServiceAuth( + self, sshPrivateKey, sshHostKey, sshKnownHosts, git_credentials + ) if self.workdir is None: self.workdir = 'gitpoller-work' @@ -600,6 +619,7 @@ async def _dovccmd( args: list[str], path: str | None = None, auth_files_path: str | None = None, + initial_stdin: str | None = None, ) -> str: full_args: list[str] = [] full_env = os.environ.copy() @@ -623,6 +643,7 @@ async def _dovccmd( [self.gitbin] + full_args, path, env=full_env, + initial_stdin=unicode2bytes(initial_stdin) if initial_stdin is not None else None, ) (code, stdout, stderr) = res stdout = bytes2unicode(stdout, self.encoding) diff --git a/master/buildbot/test/unit/changes/test_gitpoller.py b/master/buildbot/test/unit/changes/test_gitpoller.py index 25c4ec56eac2..7ee151eda14c 100644 --- a/master/buildbot/test/unit/changes/test_gitpoller.py +++ b/master/buildbot/test/unit/changes/test_gitpoller.py @@ -39,6 +39,7 @@ from buildbot.test.util.git_repository import TestGitRepository from buildbot.util import bytes2unicode from buildbot.util import unicode2bytes +from buildbot.util.git_credential import GitCredentialOptions from buildbot.util.twisted import async_to_deferred # Test that environment variables get propagated to subprocesses (See #2116) @@ -2353,6 +2354,77 @@ def test_poll_initial_2_10(self, write_local_file_mock, temp_dir_mock): self.assertEqual(expected_file_writes, write_local_file_mock.call_args_list) +class TestGitPollerWithAuthCredentials(TestGitPollerBase): + def createPoller(self): + return gitpoller.GitPoller( + self.REPOURL, + branches=['master'], + auth_credentials=('username', 'token'), + git_credentials=GitCredentialOptions( + credentials=[], + ), + ) + + @mock.patch( + 'buildbot.util.private_tempdir.PrivateTemporaryDirectory', + new_callable=MockPrivateTemporaryDirectory, + ) + @defer.inlineCallbacks + def test_poll_initial_2_10(self, temp_dir_mock): + self.expect_commands( + ExpectMasterShell(['git', '--version']).stdout(b'git version 2.10.0\n'), + ExpectMasterShell(['git', 'init', '--bare', self.POLLER_WORKDIR]), + ExpectMasterShell([ + 'git', + '-c', + 'credential.helper=', + '-c', + 'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"', + 'credential', + 'approve', + ]).workdir('basedir/gitpoller-work/.buildbot-ssh@@@'), + ExpectMasterShell([ + 'git', + '-c', + 'credential.helper=', + '-c', + 'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"', + 'ls-remote', + '--refs', + self.REPOURL, + 'refs/heads/master', + ]).stdout(b'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5\trefs/heads/master\n'), + ExpectMasterShell([ + 'git', + '-c', + 'credential.helper=', + '-c', + 'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"', + 'fetch', + '--progress', + self.REPOURL, + f'+refs/heads/master:refs/buildbot/{self.REPOURL_QUOTED}/heads/master', + '--', + ]).workdir(self.POLLER_WORKDIR), + ExpectMasterShell([ + 'git', + 'rev-parse', + f'refs/buildbot/{self.REPOURL_QUOTED}/heads/master', + ]) + .workdir(self.POLLER_WORKDIR) + .stdout(b'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5\n'), + ) + + self.poller.doPoll.running = True + yield self.poller.poll() + + self.assert_all_commands_ran() + yield self.assert_last_rev({'master': 'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5'}) + + temp_dir_path = os.path.join('basedir', 'gitpoller-work', '.buildbot-ssh@@@') + self.assertEqual(temp_dir_mock.dirs, [(temp_dir_path, 0o700)]) + + class TestGitPollerConstructor( unittest.TestCase, TestReactorMixin, changesource.ChangeSourceMixin, config.ConfigErrorsMixin ): diff --git a/master/buildbot/util/git.py b/master/buildbot/util/git.py index fac80e97d8ed..fe8fa09c8cca 100644 --- a/master/buildbot/util/git.py +++ b/master/buildbot/util/git.py @@ -37,8 +37,8 @@ from buildbot.util.twisted import async_to_deferred if TYPE_CHECKING: + from buildbot.changes.gitpoller import GitPoller from buildbot.interfaces import IRenderable - from buildbot.util.service import BuildbotService RC_SUCCESS = 0 @@ -669,14 +669,15 @@ async def remove_auth_files_if_needed(self, workdir: str) -> int: class GitServiceAuth(AbstractGitAuth): def __init__( self, - service: BuildbotService, + service: GitPoller, ssh_private_key: IRenderable | None = None, ssh_host_key: IRenderable | None = None, ssh_known_hosts: IRenderable | None = None, + git_credential_options: GitCredentialOptions | None = None, ) -> None: self._service = service - super().__init__(ssh_private_key, ssh_host_key, ssh_known_hosts) + super().__init__(ssh_private_key, ssh_host_key, ssh_known_hosts, git_credential_options) @property def _path_module(self): @@ -687,6 +688,21 @@ def _master(self): assert self._service.master is not None return self._service.master + @async_to_deferred + async def _dovccmd( + self, + command: list[str], + initial_stdin: str | None = None, + workdir: str | None = None, + ) -> None: + await self._service._dovccmd( + command=command[0], + args=command[1:], + initial_stdin=initial_stdin, + path=workdir, + auth_files_path=workdir, # this is ... not great + ) + @async_to_deferred async def _download_file( self, diff --git a/master/docs/manual/configuration/changesources.rst b/master/docs/manual/configuration/changesources.rst index 63431684ef43..9cda94d0eb6a 100755 --- a/master/docs/manual/configuration/changesources.rst +++ b/master/docs/manual/configuration/changesources.rst @@ -967,6 +967,16 @@ It accepts the following arguments: `sshPrivateKey` must be specified in order to use this option. `sshHostKey` must not be specified in order to use this option. +``auth_credentials`` + + (optional) An username/password tuple to use when running git for fetch operations. + The worker's git version needs to be at least 1.7.9. + +``git_credentials`` + + (optional) See :ref:`GitCredentialOptions`. + The worker's git version needs to be at least 1.7.9. + A configuration for the Git poller might look like this: .. code-block:: python diff --git a/newsfragments/gitpoller-credential-auth.feature b/newsfragments/gitpoller-credential-auth.feature new file mode 100644 index 000000000000..acefa8b4222a --- /dev/null +++ b/newsfragments/gitpoller-credential-auth.feature @@ -0,0 +1 @@ +:bb:chsrc:`GitPoller` now supports authentication with username/password. Credentials can be provided through the `auth_credentials` and/or `git_credentials` parameters. \ No newline at end of file