Squashed commit of the following:

commit 30f0bf5 Author: Hannah Stepanek <[email protected]> Date: Fri Jun 9 16:12:09 2023 -0700 Add OTLP protocol class & protos (#821) * Add protos under packages for otlp * Add common otlp proto payload methods * Add new oltp protocol class * Remove ML event from log message * Remove params, add api-key header & expose path The params are not relevant to OTLP so remove these. The api-key header is how we provide the license key to OTLP so add this. The path to upload dimensional metrics and events are different in OTLP so expose the path so it can be overriden inside the coresponding data_collector methods. * Add otlp_port and otlp_host settings * Default to JSON if protobuf not available & warn * Move otlp_utils to core * Call encode in protocol class * Patch issues with data collector * Move resource to utils & add log proto imports --------- Co-authored-by: Tim Pansino <[email protected]> commit e970884 Author: Timothy Pansino <[email protected]> Date: Thu Jun 8 13:17:28 2023 -0700 Dimensional Metrics (#815) * Wiring dimensional metrics * Squashed commit of the following: commit c2d4629 Author: Timothy Pansino <[email protected]> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a963649 Author: Hannah Stepanek <[email protected]> Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d82845 Author: Lalleh Rafeei <[email protected]> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei <[email protected]> Co-authored-by: Hannah Stepanek <[email protected]> commit fd0fa35 Author: Uma Annamalai <[email protected]> Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3d Author: Lalleh Rafeei <[email protected]> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506 Author: Hannah Stepanek <[email protected]> Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a Author: Hannah Stepanek <[email protected]> Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See pypa/pip#11358. * Temporarily disable kafka tests * Add dimensional stats table to stats engine * Add attribute processing to metric identity * Add testing for dimensional metrics * Cover tags as list not dict * Commit suggestions from code review
newrelic · Jun 12, 2023 · 3199cc8 · 3199cc8
1 parent 14e5a45
commit 3199cc8
Show file tree

Hide file tree

Showing 7 changed files with 167 additions and 25 deletions.
diff --git a/newrelic/config.py b/newrelic/config.py
@@ -320,6 +320,8 @@ def _process_configuration(section):
     _process_setting(section, "api_key", "get", None)
     _process_setting(section, "host", "get", None)
     _process_setting(section, "port", "getint", None)
+    _process_setting(section, "otlp_host", "get", None)
+    _process_setting(section, "otlp_port", "getint", None)
     _process_setting(section, "ssl", "getboolean", None)
     _process_setting(section, "proxy_scheme", "get", None)
     _process_setting(section, "proxy_host", "get", None)

diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py
@@ -38,14 +38,15 @@
     global_settings_dump,
 )
 from newrelic.core.internal_metrics import internal_count_metric
+from newrelic.core.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode
 from newrelic.network.exceptions import (
     DiscardDataForRequest,
     ForceAgentDisconnect,
     ForceAgentRestart,
     NetworkInterfaceException,
     RetryDataForRequest,
 )
-from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, Resource, create_key_values_from_iterable, otlp_encode
+from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode
 
 _logger = logging.getLogger(__name__)
 
@@ -528,31 +529,15 @@ def connect(
 
 
 class OtlpProtocol(AgentProtocol):
-    def __init__(self, settings, host=None, resource=None, client_cls=ApplicationModeClient):
-        self.HOST_MAP = {
-            "collector.newrelic.com": "otlp.nr-data.net",
-            "collector.eu.newrelic.com": "otlp.eu01.nr-data.net",
-            "gov-collector.newrelic.com": "gov-otlp.nr-data.net",
-            "staging-collector.newrelic.com": "staging-otlp.nr-data.net",
-            "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net",
-            "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net",
-            "fake-collector.newrelic.com": "fake-otlp.nr-data.net",
-        }
-
+    def __init__(self, settings, host=None, client_cls=ApplicationModeClient):
         if settings.audit_log_file:
             audit_log_fp = open(settings.audit_log_file, "a")
         else:
             audit_log_fp = None
 
-        otlp_host = self.HOST_MAP.get(host or settings.host, None)
-        if not otlp_host:
-            default = self.HOST_MAP["collector.newrelic.com"]
-            _logger.warn("Unable to find corresponding OTLP host using default %s" % default)
-            otlp_host = default
-
         self.client = client_cls(
-            host=otlp_host,
-            port=4318,
+            host=host or settings.otlp_host,
+            port=settings.otlp_port or 4318,
             proxy_scheme=settings.proxy_scheme,
             proxy_host=settings.proxy_host,
             proxy_port=settings.proxy_port,
@@ -561,19 +546,18 @@ def __init__(self, settings, host=None, resource=None, client_cls=ApplicationMod
             timeout=settings.agent_limits.data_collector_timeout,
             ca_bundle_path=settings.ca_bundle_path,
             disable_certificate_validation=settings.debug.disable_certificate_validation,
-            default_content_encoding_header=None,
             compression_threshold=settings.agent_limits.data_compression_threshold,
             compression_level=settings.agent_limits.data_compression_level,
             compression_method=settings.compressed_content_encoding,
             max_payload_size_in_bytes=1000000,
             audit_log_fp=audit_log_fp,
+            default_content_encoding_header=None,
         )
 
         self._params = {}
         self._headers = {
             "api-key": settings.license_key,
         }
-        self._resource = resource
 
         # In Python 2, the JSON is loaded with unicode keys and values;
         # however, the header name must be a non-unicode value when given to
@@ -606,9 +590,7 @@ def connect(
         settings,
         client_cls=ApplicationModeClient,
     ):
-        resource = Resource(attributes=create_key_values_from_iterable({"service.name": app_name}))
-
-        with cls(settings, resource=resource, client_cls=client_cls) as protocol:
+        with cls(settings, client_cls=client_cls) as protocol:
             pass
 
         return protocol

diff --git a/newrelic/core/config.py b/newrelic/core/config.py
@@ -104,6 +104,7 @@ def create_settings(nested):
 
 class TopLevelSettings(Settings):
     _host = None
+    _otlp_host = None
 
     @property
     def host(self):
@@ -115,6 +116,16 @@ def host(self):
     def host(self, value):
         self._host = value
 
+    @property
+    def otlp_host(self):
+        if self._otlp_host:
+            return self._otlp_host
+        return default_otlp_host(self.host)
+
+    @otlp_host.setter
+    def otlp_host(self, value):
+        self._otlp_host = value
+
 
 class AttributesSettings(Settings):
     pass
@@ -560,6 +571,24 @@ def default_host(license_key):
     return host
 
 
+def default_otlp_host(host):
+    HOST_MAP = {
+        "collector.newrelic.com": "otlp.nr-data.net",
+        "collector.eu.newrelic.com": "otlp.eu01.nr-data.net",
+        "gov-collector.newrelic.com": "gov-otlp.nr-data.net",
+        "staging-collector.newrelic.com": "staging-otlp.nr-data.net",
+        "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net",
+        "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net",
+        "fake-collector.newrelic.com": "fake-otlp.nr-data.net",
+    }
+    otlp_host = HOST_MAP.get(host, None)
+    if not otlp_host:
+        default = HOST_MAP["collector.newrelic.com"]
+        _logger.warn("Unable to find corresponding OTLP host using default %s" % default)
+        otlp_host = default
+    return otlp_host
+
+
 _LOG_LEVEL = {
     "CRITICAL": logging.CRITICAL,
     "ERROR": logging.ERROR,
@@ -585,7 +614,9 @@ def default_host(license_key):
 _settings.ssl = _environ_as_bool("NEW_RELIC_SSL", True)
 
 _settings.host = os.environ.get("NEW_RELIC_HOST")
+_settings.otlp_host = os.environ.get("NEW_RELIC_OTLP_HOST")
 _settings.port = int(os.environ.get("NEW_RELIC_PORT", "0"))
+_settings.otlp_port = int(os.environ.get("NEW_RELIC_OTLP_PORT", "0"))
 
 _settings.agent_run_id = None
 _settings.entity_guid = None

diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py
@@ -36,6 +36,8 @@
 
 _logger = logging.getLogger(__name__)
 
+DIMENSIONAL_METRIC_DATA_TEMP = []  # TODO: REMOVE THIS
+
 
 class Session(object):
     PROTOCOL = AgentProtocol

diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py
@@ -0,0 +1,107 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module provides common utilities for interacting with OTLP protocol buffers."""
+
+import logging
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue
+    from newrelic.packages.opentelemetry_proto.logs_pb2 import (
+        LogRecord,
+        ResourceLogs,
+        ScopeLogs,
+    )
+    from newrelic.packages.opentelemetry_proto.metrics_pb2 import (
+        AggregationTemporality,
+        Metric,
+        MetricsData,
+        NumberDataPoint,
+        ResourceMetrics,
+        ScopeMetrics,
+        Sum,
+        Summary,
+        SummaryDataPoint,
+    )
+    from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource
+
+    AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA
+    ValueAtQuantile = SummaryDataPoint.ValueAtQuantile
+
+    otlp_encode = lambda payload: payload.SerializeToString()
+    OTLP_CONTENT_TYPE = "application/x-protobuf"
+
+except ImportError:
+    from newrelic.common.encoding_utils import json_encode
+
+    def otlp_encode(*args, **kwargs):
+        _logger.warn(
+            "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss."
+        )
+        return json_encode(*args, **kwargs)
+
+    Resource = dict
+    ValueAtQuantile = dict
+    AnyValue = dict
+    KeyValue = dict
+    NumberDataPoint = dict
+    SummaryDataPoint = dict
+    Sum = dict
+    Summary = dict
+    Metric = dict
+    MetricsData = dict
+    ScopeMetrics = dict
+    ResourceMetrics = dict
+    AGGREGATION_TEMPORALITY_DELTA = 1
+    ResourceLogs = dict
+    ScopeLogs = dict
+    LogRecord = dict
+    OTLP_CONTENT_TYPE = "application/json"
+
+
+def create_key_value(key, value):
+    if isinstance(value, bool):
+        return KeyValue(key=key, value=AnyValue(bool_value=value))
+    elif isinstance(value, int):
+        return KeyValue(key=key, value=AnyValue(int_value=value))
+    elif isinstance(value, float):
+        return KeyValue(key=key, value=AnyValue(double_value=value))
+    elif isinstance(value, str):
+        return KeyValue(key=key, value=AnyValue(string_value=value))
+    # Technically AnyValue accepts array, kvlist, and bytes however, since
+    # those are not valid custom attribute types according to our api spec,
+    # we will not bother to support them here either.
+    else:
+        _logger.warn("Unsupported attribute value type %s: %s." % (key, value))
+
+
+def create_key_values_from_iterable(iterable):
+    if isinstance(iterable, dict):
+        iterable = iterable.items()
+
+    # The create_key_value list may return None if the value is an unsupported type
+    # so filter None values out before returning.
+    return list(
+        filter(
+            lambda i: i is not None,
+            (create_key_value(key, value) for key, value in iterable),
+        )
+    )
+
+
+def create_resource(attributes=None):
+    attributes = attributes or {"instrumentation.provider": "nr_performance_monitoring"}
+    return Resource(attributes=create_key_values_from_iterable(attributes))
diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py
@@ -577,6 +577,8 @@ def test_translate_deprecated_ignored_params_with_new_setting():
         ("agent_run_id", None),
         ("entity_guid", None),
         ("distributed_tracing.exclude_newrelic_header", False),
+        ("otlp_host", "otlp.nr-data.net"),
+        ("otlp_port", 0),
     ),
 )
 def test_default_values(name, expected_value):

diff --git a/tests/agent_unittests/test_utilization_settings.py b/tests/agent_unittests/test_utilization_settings.py
@@ -118,6 +118,22 @@ def reset(wrapped, instance, args, kwargs):
     return reset
 
 
+@reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF)
+def test_otlp_host_port_default():
+    settings = global_settings()
+    assert settings.otlp_host == "otlp.nr-data.net"
+    assert settings.otlp_port == 0
+
+
+@reset_agent_config(
+    INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_OTLP_HOST": "custom-otlp.nr-data.net", "NEW_RELIC_OTLP_PORT": 443}
+)
+def test_otlp_port_override():
+    settings = global_settings()
+    assert settings.otlp_host == "custom-otlp.nr-data.net"
+    assert settings.otlp_port == 443
+
+
 @reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF)
 def test_heroku_default():
     settings = global_settings()