gitpoller: implement git auth with user/password

master/buildbot/changes/gitpoller.py

@@ -30,16 +30,21 @@
 from buildbot.util import giturlparse
 from buildbot.util import private_tempdir
 from buildbot.util import runprocess
+from buildbot.util import unicode2bytes
 from buildbot.util.git import GitMixin
 from buildbot.util.git import GitServiceAuth
 from buildbot.util.git import check_ssh_config
+from buildbot.util.git_credential import GitCredentialOptions
+from buildbot.util.git_credential import add_user_password_to_credentials
 from buildbot.util.state import StateMixin
 from buildbot.util.twisted import async_to_deferred
 
 if TYPE_CHECKING:
     from typing import Callable
     from typing import Literal
 
+    from buildbot.interfaces import IRenderable
+
 
 class GitError(Exception):
     """Raised when git exits with code 128."""
@@ -97,6 +102,8 @@ def checkConfig(  # type: ignore[override]
         sshKnownHosts=None,
         pollRandomDelayMin=0,
         pollRandomDelayMax=0,
+        auth_credentials: tuple[IRenderable | str, IRenderable | str] | None = None,
+        git_credentials: GitCredentialOptions | None = None,
     ):
         if only_tags and (branch or branches):
             config.error("GitPoller: can't specify only_tags and branch/branches")
@@ -156,6 +163,8 @@ def reconfigService(
         sshKnownHosts=None,
         pollRandomDelayMin=0,
         pollRandomDelayMax=0,
+        auth_credentials: tuple[IRenderable | str, IRenderable | str] | None = None,
+        git_credentials: GitCredentialOptions | None = None,
     ):
         if name is None:
             name = repourl
@@ -186,7 +195,17 @@ def reconfigService(
         self.lastRev = None
 
         self.setupGit()
-        self._git_auth = GitServiceAuth(self, sshPrivateKey, sshHostKey, sshKnownHosts)
+
+        if auth_credentials is not None:
+            git_credentials = add_user_password_to_credentials(
+                auth_credentials,
+                repourl,
+                git_credentials,
+            )
+
+        self._git_auth = GitServiceAuth(
+            self, sshPrivateKey, sshHostKey, sshKnownHosts, git_credentials
+        )
 
         if self.workdir is None:
             self.workdir = 'gitpoller-work'
@@ -600,6 +619,7 @@ async def _dovccmd(
         args: list[str],
         path: str | None = None,
         auth_files_path: str | None = None,
+        initial_stdin: str | None = None,
     ) -> str:
         full_args: list[str] = []
         full_env = os.environ.copy()
@@ -623,6 +643,7 @@ async def _dovccmd(
             [self.gitbin] + full_args,
             path,
             env=full_env,
+            initial_stdin=unicode2bytes(initial_stdin) if initial_stdin is not None else None,
         )
         (code, stdout, stderr) = res
         stdout = bytes2unicode(stdout, self.encoding)

master/buildbot/test/unit/changes/test_gitpoller.py

@@ -39,6 +39,7 @@
 from buildbot.test.util.git_repository import TestGitRepository
 from buildbot.util import bytes2unicode
 from buildbot.util import unicode2bytes
+from buildbot.util.git_credential import GitCredentialOptions
 from buildbot.util.twisted import async_to_deferred
 
 # Test that environment variables get propagated to subprocesses (See #2116)
@@ -2353,6 +2354,77 @@ def test_poll_initial_2_10(self, write_local_file_mock, temp_dir_mock):
         self.assertEqual(expected_file_writes, write_local_file_mock.call_args_list)
 
 
+class TestGitPollerWithAuthCredentials(TestGitPollerBase):
+    def createPoller(self):
+        return gitpoller.GitPoller(
+            self.REPOURL,
+            branches=['master'],
+            auth_credentials=('username', 'token'),
+            git_credentials=GitCredentialOptions(
+                credentials=[],
+            ),
+        )
+
+    @mock.patch(
+        'buildbot.util.private_tempdir.PrivateTemporaryDirectory',
+        new_callable=MockPrivateTemporaryDirectory,
+    )
+    @defer.inlineCallbacks
+    def test_poll_initial_2_10(self, temp_dir_mock):
+        self.expect_commands(
+            ExpectMasterShell(['git', '--version']).stdout(b'git version 2.10.0\n'),
+            ExpectMasterShell(['git', 'init', '--bare', self.POLLER_WORKDIR]),
+            ExpectMasterShell([
+                'git',
+                '-c',
+                'credential.helper=',
+                '-c',
+                'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"',
+                'credential',
+                'approve',
+            ]).workdir('basedir/gitpoller-work/.buildbot-ssh@@@'),
+            ExpectMasterShell([
+                'git',
+                '-c',
+                'credential.helper=',
+                '-c',
+                'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"',
+                'ls-remote',
+                '--refs',
+                self.REPOURL,
+                'refs/heads/master',
+            ]).stdout(b'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5\trefs/heads/master\n'),
+            ExpectMasterShell([
+                'git',
+                '-c',
+                'credential.helper=',
+                '-c',
+                'credential.helper=store "--file=basedir/gitpoller-work/.buildbot-ssh@@@/.git-credentials"',
+                'fetch',
+                '--progress',
+                self.REPOURL,
+                f'+refs/heads/master:refs/buildbot/{self.REPOURL_QUOTED}/heads/master',
+                '--',
+            ]).workdir(self.POLLER_WORKDIR),
+            ExpectMasterShell([
+                'git',
+                'rev-parse',
+                f'refs/buildbot/{self.REPOURL_QUOTED}/heads/master',
+            ])
+            .workdir(self.POLLER_WORKDIR)
+            .stdout(b'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5\n'),
+        )
+
+        self.poller.doPoll.running = True
+        yield self.poller.poll()
+
+        self.assert_all_commands_ran()
+        yield self.assert_last_rev({'master': 'bf0b01df6d00ae8d1ffa0b2e2acbe642a6cd35d5'})
+
+        temp_dir_path = os.path.join('basedir', 'gitpoller-work', '.buildbot-ssh@@@')
+        self.assertEqual(temp_dir_mock.dirs, [(temp_dir_path, 0o700)])
+
+
 class TestGitPollerConstructor(
     unittest.TestCase, TestReactorMixin, changesource.ChangeSourceMixin, config.ConfigErrorsMixin
 ):

master/buildbot/util/git.py

@@ -37,8 +37,8 @@
 from buildbot.util.twisted import async_to_deferred
 
 if TYPE_CHECKING:
+    from buildbot.changes.gitpoller import GitPoller
     from buildbot.interfaces import IRenderable
-    from buildbot.util.service import BuildbotService
 
 RC_SUCCESS = 0
 
@@ -669,14 +669,15 @@ async def remove_auth_files_if_needed(self, workdir: str) -> int:
 class GitServiceAuth(AbstractGitAuth):
     def __init__(
         self,
-        service: BuildbotService,
+        service: GitPoller,
         ssh_private_key: IRenderable | None = None,
         ssh_host_key: IRenderable | None = None,
         ssh_known_hosts: IRenderable | None = None,
+        git_credential_options: GitCredentialOptions | None = None,
     ) -> None:
         self._service = service
 
-        super().__init__(ssh_private_key, ssh_host_key, ssh_known_hosts)
+        super().__init__(ssh_private_key, ssh_host_key, ssh_known_hosts, git_credential_options)
 
     @property
     def _path_module(self):
@@ -687,6 +688,21 @@ def _master(self):
         assert self._service.master is not None
         return self._service.master
 
+    @async_to_deferred
+    async def _dovccmd(
+        self,
+        command: list[str],
+        initial_stdin: str | None = None,
+        workdir: str | None = None,
+    ) -> None:
+        await self._service._dovccmd(
+            command=command[0],
+            args=command[1:],
+            initial_stdin=initial_stdin,
+            path=workdir,
+            auth_files_path=workdir,  # this is ... not great
+        )
+
     @async_to_deferred
     async def _download_file(
         self,

master/docs/manual/configuration/changesources.rst

@@ -967,6 +967,16 @@ It accepts the following arguments:
    `sshPrivateKey` must be specified in order to use this option.
    `sshHostKey` must not be specified in order to use this option.
 
+``auth_credentials``
+
+    (optional) An username/password tuple to use when running git for fetch operations.
+    The worker's git version needs to be at least 1.7.9.
+
+``git_credentials``
+
+    (optional) See :ref:`GitCredentialOptions`.
+    The worker's git version needs to be at least 1.7.9.
+
 A configuration for the Git poller might look like this:
 
 .. code-block:: python

newsfragments/gitpoller-credential-auth.feature

@@ -0,0 +1 @@
+:bb:chsrc:`GitPoller` now supports authentication with username/password. Credentials can be provided through the `auth_credentials` and/or `git_credentials` parameters.