Enable lint on CI and update deps (#2067)

* Fix black an isort

* change bootstrap_gen to use a list instead of dict

* Bunch of updates

* Fix build

* fix lint

* Fix docs

* Fix lint

* More fixes

* Fix lint

* fix stupid mistake

---------

Co-authored-by: Christian Hartung <[email protected]>

.pylintrc

@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-whitelist=
+extension-pkg-whitelist=cassandra
 
 # Add list of files or directories to be excluded. They should be base names, not
 # paths.
@@ -29,7 +29,7 @@ limit-inference-results=100
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=pylint.extensions.no_self_use
 
 # Pickle collected data for later comparisons.
 persistent=yes
@@ -69,7 +69,6 @@ disable=missing-docstring,
       duplicate-code,
       ungrouped-imports, # Leave this up to isort
       wrong-import-order, # Leave this up to isort
-      bad-continuation, # Leave this up to black
       line-too-long, # Leave this up to black
       exec-used,
       super-with-arguments, # temp-pylint-upgrade
@@ -81,6 +80,7 @@ disable=missing-docstring,
       invalid-overridden-method, # temp-pylint-upgrade 
       missing-module-docstring, # temp-pylint-upgrade
       import-error, # needed as a workaround as reported here: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/290
+      cyclic-import,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -268,13 +268,6 @@ max-line-length=79
 # Maximum number of lines in a module.
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no

dev-requirements.txt

@@ -1,20 +1,19 @@
-pylint==2.12.2
-flake8~=3.7
-isort~=5.6
-black>=22.1.0
-httpretty~=1.0
-mypy==0.790
-sphinx
-sphinx-rtd-theme~=0.4
-sphinx-autodoc-typehints
-pytest!=5.2.3
-pytest-cov>=2.8
-readme-renderer~=24.0
+pylint==3.0.2
+flake8==6.1.0
+isort==5.12.0
+black==22.3.0
+httpretty==1.1.4
+mypy==0.931
+sphinx==7.1.2
+sphinx-rtd-theme==2.0.0rc4
+sphinx-autodoc-typehints==1.25.2
+pytest==7.1.3
+pytest-cov==4.1.0
+readme-renderer==42.0
 bleach==4.1.0 # transient dependency for readme-renderer
-grpcio-tools==1.29.0
-mypy-protobuf>=1.23
 protobuf~=3.13
 markupsafe>=2.0.1
 codespell==2.1.0
 requests==2.31.0
 ruamel.yaml==0.17.21
+flaky==3.7.0

docs-requirements.txt

@@ -1,6 +1,6 @@
-sphinx==4.5.0
-sphinx-rtd-theme~=0.4
-sphinx-autodoc-typehints
+sphinx==7.1.2
+sphinx-rtd-theme==2.0.0rc4
+sphinx-autodoc-typehints==1.25.2
 
 # Need to install the api/sdk in the venv for autodoc. Modifying sys.path
 # doesn't work for pkg_resources.
@@ -45,11 +45,8 @@ remoulade>=0.50
 sqlalchemy>=1.0
 tornado>=5.1.1
 tortoise-orm>=0.17.0
-ddtrace>=0.34.0
 httpx>=0.18.0
 
 # indirect dependency pins
 markupsafe==2.0.1
 itsdangerous==2.0.1
-
-docutils==0.16

eachdist.ini

@@ -54,6 +54,7 @@ packages=
 [lintroots]
 extraroots=examples/*,scripts/
 subglob=*.py,tests/,test/,src/*,examples/*
+ignore=sklearn
 
 [testroots]
 extraroots=examples/*,tests/

exporter/opentelemetry-exporter-prometheus-remote-write/tests/conftest.py

@@ -3,6 +3,8 @@
 import pytest
 
 import opentelemetry.test.metrictestutil as metric_util
+
+# pylint: disable=no-name-in-module
 from opentelemetry.exporter.prometheus_remote_write import (
     PrometheusRemoteWriteMetricsExporter,
 )

exporter/opentelemetry-exporter-prometheus-remote-write/tests/test_prometheus_remote_write_exporter.py

@@ -17,6 +17,7 @@
 
 import pytest
 
+# pylint: disable=no-name-in-module
 from opentelemetry.exporter.prometheus_remote_write import (
     PrometheusRemoteWriteMetricsExporter,
 )

instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py

@@ -14,6 +14,7 @@
 
 import asyncio
 import contextlib
+import sys
 import typing
 import unittest
 import urllib.parse
@@ -116,14 +117,19 @@ def test_status_codes(self):
                     status_code=status_code,
                 )
 
+                url = f"http://{host}:{port}/test-path?query=param#foobar"
+                # if python version is < 3.8, then the url will be
+                if sys.version_info[1] < 8:
+                    url = f"http://{host}:{port}/test-path#foobar"
+
                 self.assert_spans(
                     [
                         (
                             "GET",
                             (span_status, None),
                             {
                                 SpanAttributes.HTTP_METHOD: "GET",
-                                SpanAttributes.HTTP_URL: f"http://{host}:{port}/test-path#foobar",
+                                SpanAttributes.HTTP_URL: url,
                                 SpanAttributes.HTTP_STATUS_CODE: int(
                                     status_code
                                 ),
@@ -136,7 +142,7 @@ def test_status_codes(self):
 
     def test_schema_url(self):
         with self.subTest(status_code=200):
-            host, port = self._http_request(
+            self._http_request(
                 trace_config=aiohttp_client.create_trace_config(),
                 url="/test-path?query=param#foobar",
                 status_code=200,
@@ -156,7 +162,7 @@ def test_not_recording(self):
         mock_tracer.start_span.return_value = mock_span
         with mock.patch("opentelemetry.trace.get_tracer"):
             # pylint: disable=W0612
-            host, port = self._http_request(
+            self._http_request(
                 trace_config=aiohttp_client.create_trace_config(),
                 url="/test-path?query=param#foobar",
             )

instrumentation/opentelemetry-instrumentation-aiohttp-server/src/opentelemetry/instrumentation/aiohttp_server/__init__.py

@@ -13,24 +13,24 @@
 # limitations under the License.
 
 import urllib
+from timeit import default_timer
+from typing import Dict, List, Tuple, Union
+
 from aiohttp import web
 from multidict import CIMultiDictProxy
-from timeit import default_timer
-from typing import Tuple, Dict, List, Union
 
-from opentelemetry import context, trace, metrics
+from opentelemetry import context, metrics, trace
 from opentelemetry.context import _SUPPRESS_HTTP_INSTRUMENTATION_KEY
 from opentelemetry.instrumentation.aiohttp_server.package import _instruments
 from opentelemetry.instrumentation.aiohttp_server.version import __version__
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
 from opentelemetry.instrumentation.utils import http_status_to_status_code
-from opentelemetry.propagators.textmap import Getter
 from opentelemetry.propagate import extract
-from opentelemetry.semconv.trace import SpanAttributes
+from opentelemetry.propagators.textmap import Getter
 from opentelemetry.semconv.metrics import MetricInstruments
+from opentelemetry.semconv.trace import SpanAttributes
 from opentelemetry.trace.status import Status, StatusCode
-from opentelemetry.util.http import get_excluded_urls
-from opentelemetry.util.http import remove_url_credentials
+from opentelemetry.util.http import get_excluded_urls, remove_url_credentials
 
 _duration_attrs = [
     SpanAttributes.HTTP_METHOD,
@@ -127,7 +127,7 @@ def collect_request_attributes(request: web.Request) -> Dict:
         result[SpanAttributes.HTTP_METHOD] = http_method
 
     http_host_value_list = (
-        [request.host] if type(request.host) != list else request.host
+        [request.host] if not isinstance(request.host, list) else request.host
     )
     if http_host_value_list:
         result[SpanAttributes.HTTP_SERVER_NAME] = ",".join(

instrumentation/opentelemetry-instrumentation-aiohttp-server/tests/test_aiohttp_server_integration.py

@@ -12,25 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from enum import Enum
+from http import HTTPStatus
+
+import aiohttp
 import pytest
 import pytest_asyncio
-import aiohttp
-from http import HTTPStatus
-from .utils import HTTPMethod
 
 from opentelemetry import trace as trace_api
-from opentelemetry.test.test_base import TestBase
-from opentelemetry.instrumentation.aiohttp_server import AioHttpServerInstrumentor
+from opentelemetry.instrumentation.aiohttp_server import (
+    AioHttpServerInstrumentor,
+)
 from opentelemetry.semconv.trace import SpanAttributes
+from opentelemetry.test.globals_test import reset_trace_globals
+from opentelemetry.test.test_base import TestBase
 from opentelemetry.util._importlib_metadata import entry_points
 
-from opentelemetry.test.globals_test import (
-    reset_trace_globals,
-)
 
+class HTTPMethod(Enum):
+    """HTTP methods and descriptions"""
+
+    def __repr__(self):
+        return f"{self.value}"
+
+    CONNECT = "CONNECT"
+    DELETE = "DELETE"
+    GET = "GET"
+    HEAD = "HEAD"
+    OPTIONS = "OPTIONS"
+    PATCH = "PATCH"
+    POST = "POST"
+    PUT = "PUT"
+    TRACE = "TRACE"
 
-@pytest.fixture(scope="session")
-def tracer():
+
+@pytest.fixture(name="tracer", scope="session")
+def fixture_tracer():
     test_base = TestBase()
 
     tracer_provider, memory_exporter = test_base.create_tracer_provider()
@@ -47,15 +64,14 @@ async def default_handler(request, status=200):
     return aiohttp.web.Response(status=status)
 
 
-@pytest_asyncio.fixture
-async def server_fixture(tracer, aiohttp_server):
+@pytest_asyncio.fixture(name="server_fixture")
+async def fixture_server_fixture(tracer, aiohttp_server):
     _, memory_exporter = tracer
 
     AioHttpServerInstrumentor().instrument()
 
     app = aiohttp.web.Application()
-    app.add_routes(
-        [aiohttp.web.get("/test-path", default_handler)])
+    app.add_routes([aiohttp.web.get("/test-path", default_handler)])
 
     server = await aiohttp_server(app)
     yield server, app
@@ -67,26 +83,31 @@ async def server_fixture(tracer, aiohttp_server):
 
 def test_checking_instrumentor_pkg_installed():
 
-    (instrumentor_entrypoint,) = entry_points(group="opentelemetry_instrumentor", name="aiohttp-server")
+    (instrumentor_entrypoint,) = entry_points(
+        group="opentelemetry_instrumentor", name="aiohttp-server"
+    )
     instrumentor = instrumentor_entrypoint.load()()
-    assert (isinstance(instrumentor, AioHttpServerInstrumentor))
+    assert isinstance(instrumentor, AioHttpServerInstrumentor)
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("url, expected_method, expected_status_code", [
-    ("/test-path", HTTPMethod.GET, HTTPStatus.OK),
-    ("/not-found", HTTPMethod.GET, HTTPStatus.NOT_FOUND)
-])
+@pytest.mark.parametrize(
+    "url, expected_method, expected_status_code",
+    [
+        ("/test-path", HTTPMethod.GET, HTTPStatus.OK),
+        ("/not-found", HTTPMethod.GET, HTTPStatus.NOT_FOUND),
+    ],
+)
 async def test_status_code_instrumentation(
     tracer,
     server_fixture,
     aiohttp_client,
     url,
     expected_method,
-    expected_status_code
+    expected_status_code,
 ):
     _, memory_exporter = tracer
-    server, app = server_fixture
+    server, _ = server_fixture
 
     assert len(memory_exporter.get_finished_spans()) == 0
 
@@ -98,8 +119,12 @@ async def test_status_code_instrumentation(
     [span] = memory_exporter.get_finished_spans()
 
     assert expected_method.value == span.attributes[SpanAttributes.HTTP_METHOD]
-    assert expected_status_code == span.attributes[SpanAttributes.HTTP_STATUS_CODE]
-
-    assert f"http://{server.host}:{server.port}{url}" == span.attributes[
-        SpanAttributes.HTTP_URL
-    ]
+    assert (
+        expected_status_code
+        == span.attributes[SpanAttributes.HTTP_STATUS_CODE]
+    )
+
+    assert (
+        f"http://{server.host}:{server.port}{url}"
+        == span.attributes[SpanAttributes.HTTP_URL]
+    )

instrumentation/opentelemetry-instrumentation-aiopg/src/opentelemetry/instrumentation/aiopg/aiopg_integration.py

@@ -215,6 +215,7 @@ async def __aexit__(self, exc_type, exc, tb):
 
 
 class _PoolAcquireContextManager(_ContextManager):
+    # pylint: disable=redefined-slots-in-subclass
     __slots__ = ("_coro", "_obj", "_pool")
 
     def __init__(self, coro, pool):

instrumentation/opentelemetry-instrumentation-asgi/src/opentelemetry/instrumentation/asgi/__init__.py

@@ -710,6 +710,7 @@ async def otel_send(message):
                         pass
 
                 await send(message)
+            # pylint: disable=too-many-boolean-expressions
             if (
                 not expecting_trailers
                 and message["type"] == "http.response.body"