From c8cd760e41f7a9b06cd5507bb468bbc174a51821 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 30 Nov 2023 10:48:00 +0100 Subject: [PATCH 1/5] Upgrade Twisted to Python 3.11 compat version Because Twisted 20 doesn't support Python 3.11, while 23.8 is the first that will - and also the last that will support Python 3.7, which we need to remain compatible until later this year. --- requirements/base.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 6a949e456d..4af7fd589e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,8 +6,7 @@ psycopg2==2.9.9 # requires libpq to build IPy==1.01 pyaml -twisted>=20.0.0,<21 - +twisted~=23.8.0 # last version that still supports Python 3.7 networkx==2.6.3 Pillow>3.3.2 From 5695742e318e83a815dd4b7fa1ba3c51981da3a5 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Wed, 28 Feb 2024 14:18:32 +0000 Subject: [PATCH 2/5] Make ProcessAMP.makeConnection work on Twisted>=21 --- python/nav/ipdevpoll/pool.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/python/nav/ipdevpoll/pool.py b/python/nav/ipdevpoll/pool.py index c5f459e277..edf72f6efd 100755 --- a/python/nav/ipdevpoll/pool.py +++ b/python/nav/ipdevpoll/pool.py @@ -158,11 +158,25 @@ def __init__(self, is_worker, **kwargs): self.lost_handler = None def makeConnection(self, transport): - """Called when a connection has been made to the AMP endpoint""" - if not hasattr(transport, 'getPeer'): + """Overrides the base implementation to fake the required getPeer() and + getHost() methods on the incoming process transport object, if needed ( the + base AMP class was not really designed with process pipe transports in mind, + but with IP transports). + + Process transports in Twisted<21 did not implement these methods at all, + while in Twisted>=21 they resolve to base methods that raise + `NotImplementError`. + """ + try: + transport.getPeer() + except (AttributeError, NotImplementedError): setattr(transport, 'getPeer', lambda: "peer") - if not hasattr(transport, 'getHost'): + + try: + transport.getHost() + except (AttributeError, NotImplementedError): setattr(transport, 'getHost', lambda: "host") + super(ProcessAMP, self).makeConnection(transport) def connectionLost(self, reason): From 070496084b8facd4d414b215ab80c553a82480bd Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Wed, 28 Feb 2024 14:47:03 +0000 Subject: [PATCH 3/5] Upgrade to latest pytest-twisted release Since we just upgraded Twisted, we should also upgrade our pytest-twisted dependency to ensure tests are still working. --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 3d1c617cca..6bd1a92ecd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,7 +9,7 @@ pytest-metadata<2.0.0 pytest-cov==2.7.1 pytest-selenium==2.0.1 pytest-timeout -pytest-twisted==1.12 +pytest-twisted~=1.14.0 pytidylib==0.3.2 selenium==3.141.0 whisper>=0.9.9 From 14208729354022872abf9c2effa4d87ee7a6ac0c Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 29 Feb 2024 12:27:41 +0000 Subject: [PATCH 4/5] Downgrade pytest-twisted to 1.13 1.14 apparently caused issues with hanging tests. Don't know why, because pytest-twisted does not seem to publish a proper changelog, so I have no idea what changed between 1.13 and 1.14. --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 6bd1a92ecd..cbbb04d3ee 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,7 +9,7 @@ pytest-metadata<2.0.0 pytest-cov==2.7.1 pytest-selenium==2.0.1 pytest-timeout -pytest-twisted~=1.14.0 +pytest-twisted~=1.13.0 pytidylib==0.3.2 selenium==3.141.0 whisper>=0.9.9 From 68e06414558b9ce42a5fce24811251a941eb4a59 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 29 Feb 2024 13:12:49 +0000 Subject: [PATCH 5/5] Fix incorrect/outdated type annotation This type annotation caused some tests to freeze indefinitely when upgrading from Twisted 20 to 23.8. The correct location for `Failure` is in the module `twisted.python.failure`, but it may have been incidentally available also in `twisted.internet.defer.failure` in earlier versions. --- python/nav/mibs/mibretriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/nav/mibs/mibretriever.py b/python/nav/mibs/mibretriever.py index 2dd1c1c419..fb6b04b48d 100644 --- a/python/nav/mibs/mibretriever.py +++ b/python/nav/mibs/mibretriever.py @@ -37,6 +37,7 @@ from twisted.internet import defer, reactor from twisted.internet.defer import returnValue from twisted.internet.error import TimeoutError +from twisted.python.failure import Failure from nav.Snmp import safestring from nav.ipdevpoll import ContextLogger @@ -428,7 +429,7 @@ def _result_formatter(result): return formatted_result - def _snmp_timeout_handler(failure: defer.failure.Failure): + def _snmp_timeout_handler(failure: Failure): """Transforms SnmpTimeoutErrors into "regular" TimeoutErrors""" failure.trap(SnmpTimeoutError) raise TimeoutError(failure.value)