Merge pull request #2352 from Uninett/python3.9

Fix Python 3.9 compatibility

.github/workflows/build-and-test.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.7, 3.8, 3.9]
 
     steps:
       - uses: actions/checkout@v2

Dockerfile

@@ -20,19 +20,18 @@
 # be world-readable!
 #
 #
-FROM debian:buster
+FROM debian:bullseye
 
 #### Prepare the OS base setup ###
 
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN echo 'deb-src http://deb.debian.org/debian buster main' >> /etc/apt/sources.list.d/srcpkg.list && \
-    echo 'deb-src http://security.debian.org/debian-security buster/updates main' >> /etc/apt/sources.list.d/srcpkg.list && \
-    echo 'deb-src http://deb.debian.org/debian buster-updates main' >> /etc/apt/sources.list.d/srcpkg.list
-
+RUN echo 'deb-src http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/srcpkg.list && \
+    echo 'deb-src http://security.debian.org/debian-security bullseye-security main' >> /etc/apt/sources.list.d/srcpkg.list
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
             locales \
+            python3-dbg gdb \
             sudo python3-dev python3-pip python3-virtualenv build-essential supervisor \
 	    debian-keyring debian-archive-keyring ca-certificates
 
@@ -49,15 +48,14 @@ RUN echo "${TIMEZONE}" > /etc/timezone && cp /usr/share/zoneinfo/${TIMEZONE} /et
 RUN apt-get update \
     && apt-get -y --no-install-recommends install \
        git-core \
-       libsnmp30 \
+       libsnmp40 \
        cron \
        sudo \
        inotify-tools \
        postgresql-client \
        vim \
        less \
        nbtscan \
-       python3-gammu \
        # Python package build deps: \
        libpq-dev \
        libjpeg-dev \
@@ -68,7 +66,7 @@ RUN apt-get update \
 RUN adduser --system --group --no-create-home --home=/source --shell=/bin/bash nav
 
 RUN pip3 install --upgrade 'setuptools<60' wheel && \
-    pip3 install --upgrade pip pip-tools
+    pip3 install --upgrade 'pip<22' pip-tools
 
 #################################################################################
 ### COPYing the requirements file to pip-install Python requirements may bust ###
@@ -83,7 +81,7 @@ COPY requirements.txt /
 COPY tests/requirements.txt /test-requirements.txt
 # Since we used pip3 to install pip globally, pip should now be for Python 3
 RUN pip-compile --output-file /requirements.txt.lock /requirements.txt /test-requirements.txt
-RUN pip-sync /requirements.txt.lock
+RUN pip install -r /requirements.txt.lock
 
 ARG CUSTOM_PIP=ipython
 RUN pip install ${CUSTOM_PIP}

NOTES.rst

@@ -41,6 +41,13 @@ version dependencies of related libraries:
 * :mod:`djangorestframework` >=3.12,<3.13
 * :mod:`markdown` >=3.3,<3.4
 
+To ensure NAV runs properly on Python 3.9, these dependency changes have also
+taken place:
+
+* :mod:`twisted` >=20.0.0,<21
+* :mod:`networkx` ==2.6.3
+
+
 Backwards incompatible changes
 ------------------------------
 

bin/ipdevpolld

@@ -15,7 +15,11 @@ from nav.bootstrap import bootstrap_django
 
 bootstrap_django(__file__)
 
-from nav.ipdevpoll.daemon import main
-
 if __name__ == '__main__':
+    from nav.ipdevpoll.epollreactor2 import install
+
+    install()
+
+    from nav.ipdevpoll.daemon import main
+
     main()

python/nav/ipdevpoll/epollreactor2.py

@@ -0,0 +1,76 @@
+#
+# Copyright (C) 2022 Sikt
+#
+# This file is part of Network Administration Visualized (NAV).
+#
+# NAV is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License version 3 as published by
+# the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.  You should have received a copy of the GNU General Public
+# License along with NAV. If not, see <http://www.gnu.org/licenses/>.
+#
+"""Custom epollreactor implementation for ipdevpoll.
+
+This reactor inherits Twisted's original epollrecator, but overrides the one
+part that seems incompatible with pynetsnmp, which is central to ipdevpoll.
+"""
+import errno
+import logging
+
+from twisted.internet import epollreactor
+
+_logger = logging.getLogger(__name__)
+
+
+class EPollReactor2(epollreactor.EPollReactor):
+    """A reactor that uses epoll(7), with modified handling of closed file
+    descriptors
+    """
+
+    def _remove(self, xer, primary, other, selectables, event, antievent):
+        """
+        Private method for removing a descriptor from the event loop.
+
+        It does the inverse job of _add, and also add a check in case of the fd
+        has gone away.
+
+        It overrides the inherited epollreactor functionality to ensure that file
+        descriptors closed behind our back are ignored and properly removed from the
+        reactor's internal data structures. This is needed mostly because pynetsnmp
+        adds reactor readers for file descriptors that are managed by the NET-SNMP C
+        library. There is no way for Python code close these file descriptors in a
+        controlled way, wherein they are removed from the reactor first - the
+        NET-SNMP library will close them "behind our backs", so to speak.
+
+        Attempting to unregister a closed file descriptor from the epoll object will
+        cause an OSError that the original implementation left the client to handle -
+        but this also caused the internal data structures of the reactor to become
+        inconsistent.
+        """
+        try:
+            super()._remove(xer, primary, other, selectables, event, antievent)
+        except OSError as error:
+            if error.errno == errno.EBADF:
+                fd = xer.fileno()
+                _logger.debug("removing/ignoring bad file descriptor %r", fd)
+                if fd in primary:
+                    primary.remove(fd)
+            else:
+                raise
+
+
+def install():
+    """
+    Install the epoll() reactor.
+    """
+    p = EPollReactor2()
+    from twisted.internet.main import installReactor
+
+    installReactor(p)
+
+
+__all__ = ["EPollReactor2", "install"]

python/nav/ipdevpoll/jobs.py

@@ -133,6 +133,7 @@ def _create_agentproxy(self):
 
     def _destroy_agentproxy(self):
         if self.agent:
+            self._logger.debug("Destroying agentproxy", self.agent)
             self.agent.close()
         self.agent = None
 
@@ -181,6 +182,9 @@ def _get_willing_plugins(self, plugin_classes):
                 can_handle = yield defer.maybeDeferred(cls.can_handle, self.netbox)
             except db.ResetDBConnectionError:
                 raise
+            # We very intentionally log and ignore unhandled exception here, to ensure
+            # the stability of the ipdevpoll daemon
+            # pylint: disable = broad-except
             except Exception:
                 self._logger.exception("Unhandled exception from can_handle(): %r", cls)
                 can_handle = False

python/nav/web/api/v1/views.py

@@ -482,8 +482,8 @@ class InterfaceViewSet(NAVAPIMixin, viewsets.ReadOnlyModelViewSet):
 
     Detail routes
     -------------
-    - last_used: interface/<id\>/last_used/
-    - metrics: interface/<id\>/metrics/
+    - last_used: interface/<id>/last_used/
+    - metrics: interface/<id>/metrics/
 
     Example: `/api/1/interface/?netbox=91&ifclass=trunk&ifclass=swport`
     """

requirements/base.txt

@@ -3,12 +3,13 @@
 
 asciitree==0.3.3  # optional, for naventity
 psycopg2==2.8.4  # requires libpq to build
-IPy==1.00
+IPy==1.01
 pyaml
 
-twisted>=16.6.0,<18
+twisted>=20.0.0,<21
 
-networkx>=2.2,<2.3
+
+networkx==2.6.3
 Pillow>3.3.2,<8.1
 pyrad==2.1
 sphinx==3.3.1

tests/docker/Dockerfile

@@ -17,6 +17,7 @@ RUN add-apt-repository ppa:deadsnakes/ppa && \
       curl git build-essential \
       python3.7-dbg python3.7-dev \
       python3.8-dbg python3.8-dev \
+      python3.9-dbg python3.9-dev \
       python3-pip
 
 RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list

tests/integration/conftest.py

@@ -31,9 +31,17 @@ def pytest_configure(config):
     os.environ['TARGETURL'] = "http://localhost:8000/"
     start_gunicorn()
 
+    # Bootstrap Django config
     from nav.bootstrap import bootstrap_django
 
     bootstrap_django('pytest')
+
+    # Install custom reactor for Twisted tests
+    from nav.ipdevpoll.epollreactor2 import install
+
+    install()
+
+    # Setup test environment for Django
     from django.test.utils import setup_test_environment
 
     setup_test_environment()

tests/unittests/netmap/metadata_nx_test.py

@@ -35,7 +35,7 @@ def test_metadata_contains_edge_objects(self):
                 assert type(pair) == Edge
 
     def test_node_a1_and_b1_contains_vlan_metadata(self):
-        vlans = self.netmap_graph.node[self.a]['metadata']['vlans']
+        vlans = self.netmap_graph.nodes[self.a]['metadata']['vlans']
 
         self.assertEqual(1, len(vlans))
         self.assertEqual(vlans[0][1], self.vlan__a1_b1)

tox.ini

@@ -4,7 +4,7 @@
 # based on tests/docker/Dockerfile
 
 [tox]
-envlist = {unit,integration,functional}-py{37,38}-django{22,32}, javascript, docs
+envlist = {unit,integration,functional}-py{37,38,39}-django{22,32}, javascript, docs
 skipsdist = True
 basepython = python3.7
 
@@ -17,6 +17,7 @@ markers =
 python =
     3.7: py37
     3.8: py38
+    3.9: py39
 
 [testenv]
 # Baseline test environment