diff --git a/master/buildbot/clients/tryclient.py b/master/buildbot/clients/tryclient.py index 802dd85a483d..d706d349c8c0 100644 --- a/master/buildbot/clients/tryclient.py +++ b/master/buildbot/clients/tryclient.py @@ -13,6 +13,8 @@ # # Copyright Buildbot Team Members +from __future__ import annotations + import base64 import json import os @@ -22,6 +24,7 @@ import string import sys import time +from typing import Literal from twisted.cred import credentials from twisted.internet import defer @@ -40,6 +43,7 @@ from buildbot.util import now from buildbot.util import unicode2bytes from buildbot.util.eventual import fireEventually +from buildbot.util.twisted import async_to_deferred class SourceStamp: @@ -732,7 +736,7 @@ def deliverJob(self): return self.deliver_job_pb() raise RuntimeError(f"unknown connecttype '{self.connect}', should be 'ssh' or 'pb'") - def getStatus(self): + async def getStatus(self) -> Literal[0, 1] | None: # returns a Deferred that fires when the builds have finished, and # may emit status messages while we wait wait = bool(self.getopt("wait")) @@ -741,7 +745,7 @@ def getStatus(self): elif self.connect == "ssh": output("waiting for builds with ssh is not supported") else: - self.running = defer.Deferred() + self.running: defer.Deferred[Literal[0, 1]] = defer.Deferred() if not self.buildsetStatus: output("try scheduler on the master does not have the builder configured") return None @@ -749,7 +753,7 @@ def getStatus(self): self._getStatus_1() # note that we don't wait for the returned Deferred if bool(self.config.get("dryrun")): self.statusDone() - return self.running + return await self.running return None @defer.inlineCallbacks @@ -904,33 +908,33 @@ def getAvailableBuilderNames(self): sys.exit(1) raise RuntimeError(f"unknown connecttype '{self.connect}', should be 'pb'") - def announce(self, message): + def announce(self, message) -> None: if not self.quiet: output(message) - @defer.inlineCallbacks - def run_impl(self): + @async_to_deferred + async def run_impl(self) -> None: output(f"using '{self.connect}' connect method") self.exitcode = 0 # we can't do spawnProcess until we're inside reactor.run(), so force asynchronous execution - yield fireEventually(None) + await fireEventually(None) try: if bool(self.config.get("get-builder-names")): - yield self.getAvailableBuilderNames() + await self.getAvailableBuilderNames() else: - yield self.createJob() - yield self.announce("job created") + await self.createJob() + self.announce("job created") if bool(self.config.get("dryrun")): - yield self.fakeDeliverJob() + await self.fakeDeliverJob() else: - yield self.deliverJob() - yield self.announce("job has been delivered") - yield self.getStatus() + await self.deliverJob() + self.announce("job has been delivered") + await self.getStatus() if not bool(self.config.get("dryrun")): - yield self.cleanup() + self.cleanup() except SystemExit as e: self.exitcode = e.code except Exception as e: @@ -948,6 +952,6 @@ def trapSystemExit(self, why): why.trap(SystemExit) self.exitcode = why.value.code - def cleanup(self, res=None): + def cleanup(self, res=None) -> None: if self.buildsetStatus: self.buildsetStatus.broker.transport.loseConnection() diff --git a/master/buildbot/test/integration/test_try_client.py b/master/buildbot/test/integration/test_try_client.py index d1f120ac64c5..a60eec0add9b 100644 --- a/master/buildbot/test/integration/test_try_client.py +++ b/master/buildbot/test/integration/test_try_client.py @@ -13,8 +13,14 @@ # # Copyright Buildbot Team Members +from __future__ import annotations +import inspect import os +from typing import Any +from typing import Callable +from typing import Coroutine +from typing import TypeVar from unittest import mock from twisted.internet import defer @@ -24,23 +30,36 @@ from buildbot import util from buildbot.clients import tryclient +from buildbot.master import BuildMaster from buildbot.schedulers import trysched from buildbot.test.util import www from buildbot.test.util.integration import RunMasterBase +from buildbot.util.twisted import async_to_deferred + +_T = TypeVar('_T') # wait for some asynchronous result -@defer.inlineCallbacks -def waitFor(fn): +async def waitFor(fn: Callable[[], defer.Deferred[_T] | Coroutine[Any, Any, _T] | _T]) -> None: while True: - res = yield fn() + call: defer.Deferred[_T] | Coroutine[Any, Any, _T] | _T = fn() + if inspect.isawaitable(call) or isinstance(call, defer.Deferred): + res = await call + else: + res = call + if res: - return res - yield util.asyncSleep(0.01) + break + await util.asyncSleep(0.01) + +class _TrySchedulersBase(RunMasterBase, www.RequiresWwwMixin): + output: list[str] + serverPort: str + master: BuildMaster | None + sch: trysched.TryBase | None -class Schedulers(RunMasterBase, www.RequiresWwwMixin): - def setUp(self): + def setUp(self) -> None: self.master = None self.sch = None @@ -52,6 +71,7 @@ def spawnProcess(pp, executable, args, environ): os.rename(tmpfile, newfile) log.msg(f"wrote jobfile {newfile}") # get the scheduler to poll this directory now + assert isinstance(self.sch, trysched.Try_Jobdir) d = self.sch.watcher.poll() d.addErrback(log.err, 'while polling') @@ -92,9 +112,8 @@ def setupJobdir(self): jobdir.child(sub).createDirectory() return self.jobdir - @defer.inlineCallbacks - def setup_config(self, extra_config): - c = {} + async def setup_config(self, extra_config) -> None: + c: dict[str, Any] = {} from buildbot.config import BuilderConfig from buildbot.process import results from buildbot.process.buildstep import BuildStep @@ -119,48 +138,83 @@ def run(self): # test wants to influence the config, but we still return a new config # each time c.update(extra_config) - yield self.setup_master(c) + await self.setup_master(c) - @defer.inlineCallbacks - def startMaster(self, sch): + async def startMaster(self, sch: trysched.TryBase) -> None: + assert isinstance(sch, trysched.TryBase), f"{type(sch)=}" extra_config = { 'schedulers': [sch], } self.sch = sch - yield self.setup_config(extra_config) + await self.setup_config(extra_config) # wait until the scheduler is active - yield waitFor(lambda: self.sch.active) + await waitFor(lambda: self.sch is not None and self.sch.active) # and, for Try_Userpass, until it's registered its port if isinstance(self.sch, trysched.Try_Userpass): - def getSchedulerPort(): + def getSchedulerPort() -> bool: + assert self.sch is not None and isinstance(self.sch, trysched.Try_Userpass) if not self.sch.registrations: - return None + return False self.serverPort = self.sch.registrations[0].getPort() log.msg(f"Scheduler registered at port {self.serverPort}") return True - yield waitFor(getSchedulerPort) + await waitFor(getSchedulerPort) - def runClient(self, config): + async def runClient(self, config) -> None: self.clt = tryclient.Try(config) - return self.clt.run_impl() - - @defer.inlineCallbacks - def test_userpass_no_wait(self): - yield self.startMaster(trysched.Try_Userpass('try', ['a'], 0, [('u', b'p')])) - yield self.runClient({ - 'connect': 'pb', - 'master': f'127.0.0.1:{self.serverPort}', + await self.clt.run_impl() + + async def _base_run_client( + self, + run_client_args: dict[str, Any], + expected_output: list[str], + ) -> list: + await self.runClient({ 'username': 'u', 'passwd': b'p', + **run_client_args, }) self.assertEqual( self.output, - [ + expected_output, + ) + assert self.master is not None + return await self.master.db.buildsets.getBuildsets() + + +class TrySchedulerUserPass(_TrySchedulersBase): + async def _base_run_client_userpass( + self, + run_client_args: dict[str, Any], + expected_output: list[str], + ) -> list: + await self.startMaster( + trysched.Try_Userpass( + 'try', + ['a'], + 0, + [('u', b'p')], + ) + ) + return await self._base_run_client( + run_client_args={ + 'connect': 'pb', + 'master': f'127.0.0.1:{self.serverPort}', + **run_client_args, + }, + expected_output=expected_output, + ) + + @async_to_deferred + async def test_userpass_no_wait(self): + buildsets = await self._base_run_client_userpass( + run_client_args={}, + expected_output=[ "using 'pb' connect method", 'job created', 'Delivering job; comment= None', @@ -168,22 +222,14 @@ def test_userpass_no_wait(self): 'not waiting for builds to finish', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 1) - @defer.inlineCallbacks - def test_userpass_wait(self): - yield self.startMaster(trysched.Try_Userpass('try', ['a'], 0, [('u', b'p')])) - yield self.runClient({ - 'connect': 'pb', - 'master': f'127.0.0.1:{self.serverPort}', - 'username': 'u', - 'passwd': b'p', - 'wait': True, - }) - self.assertEqual( - self.output, - [ + async def test_userpass_wait(self) -> None: + buildsets = await self._base_run_client_userpass( + run_client_args={ + 'wait': True, + }, + expected_output=[ "using 'pb' connect method", 'job created', 'Delivering job; comment= None', @@ -192,24 +238,16 @@ def test_userpass_wait(self): 'a: success (build successful)', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 1) - @defer.inlineCallbacks - def test_userpass_wait_bytes(self): + async def test_userpass_wait_bytes(self) -> None: self.sourcestamp = tryclient.SourceStamp(branch=b'br', revision=b'rr', patch=(0, b'++--')) - yield self.startMaster(trysched.Try_Userpass('try', ['a'], 0, [('u', b'p')])) - yield self.runClient({ - 'connect': 'pb', - 'master': f'127.0.0.1:{self.serverPort}', - 'username': 'u', - 'passwd': b'p', - 'wait': True, - }) - self.assertEqual( - self.output, - [ + buildsets = await self._base_run_client_userpass( + run_client_args={ + 'wait': True, + }, + expected_output=[ "using 'pb' connect method", 'job created', 'Delivering job; comment= None', @@ -218,23 +256,15 @@ def test_userpass_wait_bytes(self): 'a: success (build successful)', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 1) - @defer.inlineCallbacks - def test_userpass_wait_dryrun(self): - yield self.startMaster(trysched.Try_Userpass('try', ['a'], 0, [('u', b'p')])) - yield self.runClient({ - 'connect': 'pb', - 'master': f'127.0.0.1:{self.serverPort}', - 'username': 'u', - 'passwd': b'p', - 'wait': True, - 'dryrun': True, - }) - self.assertEqual( - self.output, - [ + async def test_userpass_wait_dryrun(self) -> None: + buildsets = await self._base_run_client_userpass( + run_client_args={ + 'wait': True, + 'dryrun': True, + }, + expected_output=[ "using 'pb' connect method", 'job created', 'Job:\n' @@ -248,73 +278,64 @@ def test_userpass_wait_dryrun(self): 'All Builds Complete', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 0) - @defer.inlineCallbacks - def test_userpass_list_builders(self): - yield self.startMaster(trysched.Try_Userpass('try', ['a'], 0, [('u', b'p')])) - yield self.runClient({ - 'connect': 'pb', - 'get-builder-names': True, - 'master': f'127.0.0.1:{self.serverPort}', - 'username': 'u', - 'passwd': b'p', - }) - self.assertEqual( - self.output, - [ + async def test_userpass_list_builders(self) -> None: + buildsets = await self._base_run_client_userpass( + run_client_args={ + 'get-builder-names': True, + }, + expected_output=[ "using 'pb' connect method", 'The following builders are available for the try scheduler: ', 'a', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 0) - @defer.inlineCallbacks - def test_jobdir_no_wait(self): + +class TrySchedulerJobDir(_TrySchedulersBase): + async def _base_run_client_jobdir( + self, + run_client_args: dict[str, Any], + expected_output: list[str], + ) -> list: jobdir = self.setupJobdir() - yield self.startMaster(trysched.Try_Jobdir('try', ['a'], jobdir)) - yield self.runClient({ - 'connect': 'ssh', - 'master': '127.0.0.1', - 'username': 'u', - 'passwd': b'p', - 'builders': 'a', # appears to be required for ssh - }) - self.assertEqual( - self.output, - [ + await self.startMaster(trysched.Try_Jobdir('try', ['a'], jobdir)) + return await self._base_run_client( + run_client_args={ + 'connect': 'ssh', + 'master': '127.0.0.1', + **run_client_args, + }, + expected_output=expected_output, + ) + + async def test_jobdir_no_wait(self) -> None: + buildsets = await self._base_run_client_jobdir( + run_client_args={ + 'builders': 'a', # appears to be required for ssh + }, + expected_output=[ "using 'ssh' connect method", 'job created', 'job has been delivered', 'not waiting for builds to finish', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 1) - @defer.inlineCallbacks - def test_jobdir_wait(self): - jobdir = self.setupJobdir() - yield self.startMaster(trysched.Try_Jobdir('try', ['a'], jobdir)) - yield self.runClient({ - 'connect': 'ssh', - 'wait': True, - 'host': '127.0.0.1', - 'username': 'u', - 'passwd': b'p', - 'builders': 'a', # appears to be required for ssh - }) - self.assertEqual( - self.output, - [ + async def test_jobdir_wait(self) -> None: + buildsets = await self._base_run_client_jobdir( + run_client_args={ + 'wait': True, + 'builders': 'a', # appears to be required for ssh + }, + expected_output=[ "using 'ssh' connect method", 'job created', 'job has been delivered', 'waiting for builds with ssh is not supported', ], ) - buildsets = yield self.master.db.buildsets.getBuildsets() self.assertEqual(len(buildsets), 1) diff --git a/master/buildbot/test/util/integration.py b/master/buildbot/test/util/integration.py index 58c88b8171c1..0211ba98cd52 100644 --- a/master/buildbot/test/util/integration.py +++ b/master/buildbot/test/util/integration.py @@ -37,9 +37,9 @@ from buildbot.test.reactor import TestReactorMixin from buildbot.test.util.misc import DebugIntegrationLogsMixin from buildbot.test.util.sandboxed_worker import SandboxedWorker -from buildbot.worker.local import LocalWorker from buildbot.util.twisted import any_to_async from buildbot.util.twisted import async_to_deferred +from buildbot.worker.local import LocalWorker Worker: type | None = None try: