Skip to content

Commit

Permalink
Add exclude_list for urllib/urllib3 instrumentations (#1733)
Browse files Browse the repository at this point in the history
* urllib

* urllib3

* Update __init__.py

* readme

* lint
  • Loading branch information
lzchen authored Mar 31, 2023
1 parent 5052190 commit 4e059b1
Show file tree
Hide file tree
Showing 8 changed files with 217 additions and 6 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
([#1690](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1690))
- Add metrics instrumentation for sqlalchemy
([#1645](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1645))
- Add `excluded_urls` functionality to `urllib` and `urllib3` instrumentations
([#1733](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1733))

### Fixed

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import functools
import types
from timeit import default_timer
from typing import Callable, Collection, Iterable, Optional
from typing import Callable, Collection, Optional
from urllib.parse import urlparse

from requests.models import PreparedRequest, Response
Expand All @@ -77,6 +77,7 @@
from opentelemetry.trace.span import Span
from opentelemetry.trace.status import Status
from opentelemetry.util.http import (
ExcludeList,
get_excluded_urls,
parse_excluded_urls,
remove_url_credentials,
Expand All @@ -96,7 +97,7 @@ def _instrument(
duration_histogram: Histogram,
request_hook: _RequestHookT = None,
response_hook: _ResponseHookT = None,
excluded_urls: Iterable[str] = None,
excluded_urls: ExcludeList = None,
):
"""Enables tracing of all requests calls that go through
:code:`requests.session.Session.request` (this includes
Expand Down
40 changes: 40 additions & 0 deletions instrumentation/opentelemetry-instrumentation-urllib/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,46 @@ Installation

pip install opentelemetry-instrumentation-urllib

Configuration
-------------

Request/Response hooks
**********************

The urllib instrumentation supports extending tracing behavior with the help of
request and response hooks. These are functions that are called back by the instrumentation
right after a Span is created for a request and right before the span is finished processing a response respectively.
The hooks can be configured as follows:

.. code:: python
# `request_obj` is an instance of urllib.request.Request
def request_hook(span, request_obj):
pass
# `request_obj` is an instance of urllib.request.Request
# `response` is an instance of http.client.HTTPResponse
def response_hook(span, request_obj, response)
pass
URLLibInstrumentor.instrument(
request_hook=request_hook, response_hook=response_hook)
)
Exclude lists
*************

To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_URLLIB_EXCLUDED_URLS``
(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude.

For example,

::

export OTEL_PYTHON_URLLIB_EXCLUDED_URLS="client/.*/info,healthcheck"

will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.

References
----------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,20 @@ def response_hook(span, request_obj, response)
request_hook=request_hook, response_hook=response_hook)
)
Exclude lists
*************
To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_URLLIB_EXCLUDED_URLS``
(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude.
For example,
::
export OTEL_PYTHON_URLLIB_EXCLUDED_URLS="client/.*/info,healthcheck"
will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.
API
---
"""
Expand Down Expand Up @@ -88,7 +102,14 @@ def response_hook(span, request_obj, response)
from opentelemetry.semconv.trace import SpanAttributes
from opentelemetry.trace import Span, SpanKind, get_tracer
from opentelemetry.trace.status import Status
from opentelemetry.util.http import remove_url_credentials
from opentelemetry.util.http import (
ExcludeList,
get_excluded_urls,
parse_excluded_urls,
remove_url_credentials,
)

_excluded_urls_from_env = get_excluded_urls("URLLIB")

_RequestHookT = typing.Optional[typing.Callable[[Span, Request], None]]
_ResponseHookT = typing.Optional[
Expand All @@ -112,10 +133,12 @@ def _instrument(self, **kwargs):
``tracer_provider``: a TracerProvider, defaults to global
``request_hook``: An optional callback invoked that is invoked right after a span is created.
``response_hook``: An optional callback which is invoked right before the span is finished processing a response
``excluded_urls``: A string containing a comma-delimited
list of regexes used to exclude URLs from tracking
"""
tracer_provider = kwargs.get("tracer_provider")
tracer = get_tracer(__name__, __version__, tracer_provider)

excluded_urls = kwargs.get("excluded_urls")
meter_provider = kwargs.get("meter_provider")
meter = get_meter(__name__, __version__, meter_provider)

Expand All @@ -126,6 +149,9 @@ def _instrument(self, **kwargs):
histograms,
request_hook=kwargs.get("request_hook"),
response_hook=kwargs.get("response_hook"),
excluded_urls=_excluded_urls_from_env
if excluded_urls is None
else parse_excluded_urls(excluded_urls),
)

def _uninstrument(self, **kwargs):
Expand All @@ -143,6 +169,7 @@ def _instrument(
histograms: Dict[str, Histogram],
request_hook: _RequestHookT = None,
response_hook: _ResponseHookT = None,
excluded_urls: ExcludeList = None,
):
"""Enables tracing of all requests calls that go through
:code:`urllib.Client._make_request`"""
Expand Down Expand Up @@ -174,8 +201,11 @@ def _instrumented_open_call(
) or context.get_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY):
return call_wrapped()

method = request.get_method().upper()
url = request.full_url
if excluded_urls and excluded_urls.url_disabled(url):
return call_wrapped()

method = request.get_method().upper()

span_name = f"HTTP {method}".strip()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from opentelemetry.test.mock_textmap import MockTextMapPropagator
from opentelemetry.test.test_base import TestBase
from opentelemetry.trace import StatusCode
from opentelemetry.util.http import get_excluded_urls

# pylint: disable=too-many-public-methods

Expand All @@ -52,6 +53,21 @@ class RequestsIntegrationTestBase(abc.ABC):
# pylint: disable=invalid-name
def setUp(self):
super().setUp()

self.env_patch = mock.patch.dict(
"os.environ",
{
"OTEL_PYTHON_URLLIB_EXCLUDED_URLS": "http://localhost/env_excluded_arg/123,env_excluded_noarg"
},
)
self.env_patch.start()

self.exclude_patch = mock.patch(
"opentelemetry.instrumentation.urllib._excluded_urls_from_env",
get_excluded_urls("URLLIB"),
)
self.exclude_patch.start()

URLLibInstrumentor().instrument()
httpretty.enable()
httpretty.register_uri(httpretty.GET, self.URL, body=b"Hello!")
Expand Down Expand Up @@ -125,6 +141,36 @@ def test_basic(self):
span, opentelemetry.instrumentation.urllib
)

def test_excluded_urls_explicit(self):
url_201 = "http://httpbin.org/status/201"
httpretty.register_uri(
httpretty.GET,
url_201,
status=201,
)

URLLibInstrumentor().uninstrument()
URLLibInstrumentor().instrument(excluded_urls=".*/201")
self.perform_request(self.URL)
self.perform_request(url_201)

self.assert_span(num_spans=1)

def test_excluded_urls_from_env(self):
url = "http://localhost/env_excluded_arg/123"
httpretty.register_uri(
httpretty.GET,
url,
status=200,
)

URLLibInstrumentor().uninstrument()
URLLibInstrumentor().instrument()
self.perform_request(self.URL)
self.perform_request(url)

self.assert_span(num_spans=1)

def test_not_foundbasic(self):
url_404 = "http://httpbin.org/status/404/"
httpretty.register_uri(
Expand Down
14 changes: 14 additions & 0 deletions instrumentation/opentelemetry-instrumentation-urllib3/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,20 @@ The hooks can be configured as follows:
request_hook=request_hook, response_hook=response_hook)
)
Exclude lists
*************

To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_URLLIB3_EXCLUDED_URLS``
(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude.

For example,

::

export OTEL_PYTHON_URLLIB3_EXCLUDED_URLS="client/.*/info,healthcheck"

will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.

References
----------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ def response_hook(span, request, response):
request_hook=request_hook, response_hook=response_hook)
)
Exclude lists
*************
To exclude certain URLs from being tracked, set the environment variable ``OTEL_PYTHON_URLLIB3_EXCLUDED_URLS``
(or ``OTEL_PYTHON_EXCLUDED_URLS`` as fallback) with comma delimited regexes representing which URLs to exclude.
For example,
::
export OTEL_PYTHON_URLLIB3_EXCLUDED_URLS="client/.*/info,healthcheck"
will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``.
API
---
"""
Expand Down Expand Up @@ -92,8 +106,15 @@ def response_hook(span, request, response):
from opentelemetry.semconv.trace import SpanAttributes
from opentelemetry.trace import Span, SpanKind, Tracer, get_tracer
from opentelemetry.trace.status import Status
from opentelemetry.util.http import (
ExcludeList,
get_excluded_urls,
parse_excluded_urls,
)
from opentelemetry.util.http.httplib import set_ip_on_next_http_connection

_excluded_urls_from_env = get_excluded_urls("URLLIB3")

_UrlFilterT = typing.Optional[typing.Callable[[str], str]]
_RequestHookT = typing.Optional[
typing.Callable[
Expand Down Expand Up @@ -138,10 +159,14 @@ def _instrument(self, **kwargs):
``response_hook``: An optional callback which is invoked right before the span is finished processing a response.
``url_filter``: A callback to process the requested URL prior
to adding it as a span attribute.
``excluded_urls``: A string containing a comma-delimited
list of regexes used to exclude URLs from tracking
"""
tracer_provider = kwargs.get("tracer_provider")
tracer = get_tracer(__name__, __version__, tracer_provider)

excluded_urls = kwargs.get("excluded_urls")

meter_provider = kwargs.get("meter_provider")
meter = get_meter(__name__, __version__, meter_provider)

Expand Down Expand Up @@ -169,6 +194,9 @@ def _instrument(self, **kwargs):
request_hook=kwargs.get("request_hook"),
response_hook=kwargs.get("response_hook"),
url_filter=kwargs.get("url_filter"),
excluded_urls=_excluded_urls_from_env
if excluded_urls is None
else parse_excluded_urls(excluded_urls),
)

def _uninstrument(self, **kwargs):
Expand All @@ -183,13 +211,17 @@ def _instrument(
request_hook: _RequestHookT = None,
response_hook: _ResponseHookT = None,
url_filter: _UrlFilterT = None,
excluded_urls: ExcludeList = None,
):
def instrumented_urlopen(wrapped, instance, args, kwargs):
if _is_instrumentation_suppressed():
return wrapped(*args, **kwargs)

method = _get_url_open_arg("method", args, kwargs).upper()
url = _get_url(instance, args, kwargs, url_filter)
if excluded_urls and excluded_urls.url_disabled(url):
return wrapped(*args, **kwargs)

method = _get_url_open_arg("method", args, kwargs).upper()
headers = _prepare_headers(kwargs)
body = _get_url_open_arg("body", args, kwargs)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from opentelemetry.semconv.trace import SpanAttributes
from opentelemetry.test.mock_textmap import MockTextMapPropagator
from opentelemetry.test.test_base import TestBase
from opentelemetry.util.http import get_excluded_urls

# pylint: disable=too-many-public-methods

Expand All @@ -39,6 +40,21 @@ class TestURLLib3Instrumentor(TestBase):

def setUp(self):
super().setUp()

self.env_patch = mock.patch.dict(
"os.environ",
{
"OTEL_PYTHON_URLLIB3_EXCLUDED_URLS": "http://localhost/env_excluded_arg/123,env_excluded_noarg"
},
)
self.env_patch.start()

self.exclude_patch = mock.patch(
"opentelemetry.instrumentation.urllib3._excluded_urls_from_env",
get_excluded_urls("URLLIB3"),
)
self.exclude_patch.start()

URLLib3Instrumentor().instrument()

httpretty.enable(allow_net_connect=False)
Expand Down Expand Up @@ -158,6 +174,36 @@ def test_url_open_explicit_arg_parameters(self):

self.assert_success_span(response, url)

def test_excluded_urls_explicit(self):
url_201 = "http://httpbin.org/status/201"
httpretty.register_uri(
httpretty.GET,
url_201,
status=201,
)

URLLib3Instrumentor().uninstrument()
URLLib3Instrumentor().instrument(excluded_urls=".*/201")
self.perform_request(self.HTTP_URL)
self.perform_request(url_201)

self.assert_span(num_spans=1)

def test_excluded_urls_from_env(self):
url = "http://localhost/env_excluded_arg/123"
httpretty.register_uri(
httpretty.GET,
url,
status=200,
)

URLLib3Instrumentor().uninstrument()
URLLib3Instrumentor().instrument()
self.perform_request(self.HTTP_URL)
self.perform_request(url)

self.assert_span(num_spans=1)

def test_uninstrument(self):
URLLib3Instrumentor().uninstrument()

Expand Down

0 comments on commit 4e059b1

Please sign in to comment.