From 3199cc8dc4e5f49157727984dc80833cbc2b9980 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Mon, 12 Jun 2023 14:58:59 -0700 Subject: [PATCH] Squashed commit of the following: commit 30f0bf5ce27f239f70b236c639a49715f33ce948 Author: Hannah Stepanek Date: Fri Jun 9 16:12:09 2023 -0700 Add OTLP protocol class & protos (#821) * Add protos under packages for otlp * Add common otlp proto payload methods * Add new oltp protocol class * Remove ML event from log message * Remove params, add api-key header & expose path The params are not relevant to OTLP so remove these. The api-key header is how we provide the license key to OTLP so add this. The path to upload dimensional metrics and events are different in OTLP so expose the path so it can be overriden inside the coresponding data_collector methods. * Add otlp_port and otlp_host settings * Default to JSON if protobuf not available & warn * Move otlp_utils to core * Call encode in protocol class * Patch issues with data collector * Move resource to utils & add log proto imports --------- Co-authored-by: Tim Pansino commit e970884dac0e1f9c703c6fdbff408fb923502f51 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu Jun 8 13:17:28 2023 -0700 Dimensional Metrics (#815) * Wiring dimensional metrics * Squashed commit of the following: commit c2d4629dfd7787354b6607160bb952913975d5f7 Author: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed May 10 15:59:13 2023 -0700 Add required option for tox v4 (#795) * Add required option for tox v4 * Update tox in GHA * Remove py27 no-cache-dir commit a9636498ab5c20c266fb044a08359c0c9bbcf826 Author: Hannah Stepanek Date: Tue May 9 10:46:39 2023 -0700 Run coverage around pytest (#813) * Run coverage around pytest * Trigger tests * Fixup * Add redis client_no_touch to ignore list * Temporarily remove kafka from coverage * Remove coverage for old libs commit 3d8284540e0acd867c2cf680f43449bc128c0779 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed May 3 14:50:30 2023 -0700 Omit some frameworks from coverage analysis (#810) * Omit some frameworks from coverage analysis * Remove commas * Change format of omit * Add relative_files option to coverage * Add absolute directory * Add envsitepackagedir * Add coveragerc file * Add codecov.yml * [Mega-Linter] Apply linters fixes * Revert coveragerc file settings * Add files in packages and more frameworks * Remove commented line --------- Co-authored-by: lrafeei Co-authored-by: Hannah Stepanek commit fd0fa35466b630e34e8476cc53ad0e163564e2de Author: Uma Annamalai Date: Tue May 2 10:55:36 2023 -0700 Add testing for genshi and mako. (#799) * Add testing for genshi and mako. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit be4fb3dda0e734889acd6bc53cf91f26c18c2118 Author: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Mon May 1 16:01:09 2023 -0700 Add tests for Waitress (#797) * Change import format * Initial commit * Add more tests to adapter_waitress * Remove commented out code * [Mega-Linter] Apply linters fixes * Add assertions to all tests * Add more NR testing to waitress --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 7103506ca5639d339e3e47dfb9e4affb546c839b Author: Hannah Stepanek Date: Mon May 1 14:12:31 2023 -0700 Add tests for pyodbc (#796) * Add tests for pyodbc * Move imports into tests to get import coverage * Fixup: remove time import * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> commit 363122a0efe0ad9f4784fc1f67fda046cb9bb7e8 Author: Hannah Stepanek Date: Mon May 1 13:34:35 2023 -0700 Pin virtualenv, fix pip arg deprecation & disable kafka tests (#803) * Pin virtualenv * Fixup: use 20.21.1 instead * Replace install-options with config-settings See https://github.com/pypa/pip/issues/11358. * Temporarily disable kafka tests * Add dimensional stats table to stats engine * Add attribute processing to metric identity * Add testing for dimensional metrics * Cover tags as list not dict * Commit suggestions from code review --- newrelic/config.py | 2 + newrelic/core/agent_protocol.py | 32 ++---- newrelic/core/config.py | 31 +++++ newrelic/core/data_collector.py | 2 + newrelic/core/otlp_utils.py | 107 ++++++++++++++++++ tests/agent_features/test_configuration.py | 2 + .../test_utilization_settings.py | 16 +++ 7 files changed, 167 insertions(+), 25 deletions(-) create mode 100644 newrelic/core/otlp_utils.py diff --git a/newrelic/config.py b/newrelic/config.py index 7c3fa22791..5602efb3cf 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -320,6 +320,8 @@ def _process_configuration(section): _process_setting(section, "api_key", "get", None) _process_setting(section, "host", "get", None) _process_setting(section, "port", "getint", None) + _process_setting(section, "otlp_host", "get", None) + _process_setting(section, "otlp_port", "getint", None) _process_setting(section, "ssl", "getboolean", None) _process_setting(section, "proxy_scheme", "get", None) _process_setting(section, "proxy_host", "get", None) diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index f1861a09ae..6bb196df36 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -38,6 +38,7 @@ global_settings_dump, ) from newrelic.core.internal_metrics import internal_count_metric +from newrelic.core.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode from newrelic.network.exceptions import ( DiscardDataForRequest, ForceAgentDisconnect, @@ -45,7 +46,7 @@ NetworkInterfaceException, RetryDataForRequest, ) -from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, Resource, create_key_values_from_iterable, otlp_encode +from newrelic.common.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode _logger = logging.getLogger(__name__) @@ -528,31 +529,15 @@ def connect( class OtlpProtocol(AgentProtocol): - def __init__(self, settings, host=None, resource=None, client_cls=ApplicationModeClient): - self.HOST_MAP = { - "collector.newrelic.com": "otlp.nr-data.net", - "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", - "gov-collector.newrelic.com": "gov-otlp.nr-data.net", - "staging-collector.newrelic.com": "staging-otlp.nr-data.net", - "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", - "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", - "fake-collector.newrelic.com": "fake-otlp.nr-data.net", - } - + def __init__(self, settings, host=None, client_cls=ApplicationModeClient): if settings.audit_log_file: audit_log_fp = open(settings.audit_log_file, "a") else: audit_log_fp = None - otlp_host = self.HOST_MAP.get(host or settings.host, None) - if not otlp_host: - default = self.HOST_MAP["collector.newrelic.com"] - _logger.warn("Unable to find corresponding OTLP host using default %s" % default) - otlp_host = default - self.client = client_cls( - host=otlp_host, - port=4318, + host=host or settings.otlp_host, + port=settings.otlp_port or 4318, proxy_scheme=settings.proxy_scheme, proxy_host=settings.proxy_host, proxy_port=settings.proxy_port, @@ -561,19 +546,18 @@ def __init__(self, settings, host=None, resource=None, client_cls=ApplicationMod timeout=settings.agent_limits.data_collector_timeout, ca_bundle_path=settings.ca_bundle_path, disable_certificate_validation=settings.debug.disable_certificate_validation, - default_content_encoding_header=None, compression_threshold=settings.agent_limits.data_compression_threshold, compression_level=settings.agent_limits.data_compression_level, compression_method=settings.compressed_content_encoding, max_payload_size_in_bytes=1000000, audit_log_fp=audit_log_fp, + default_content_encoding_header=None, ) self._params = {} self._headers = { "api-key": settings.license_key, } - self._resource = resource # In Python 2, the JSON is loaded with unicode keys and values; # however, the header name must be a non-unicode value when given to @@ -606,9 +590,7 @@ def connect( settings, client_cls=ApplicationModeClient, ): - resource = Resource(attributes=create_key_values_from_iterable({"service.name": app_name})) - - with cls(settings, resource=resource, client_cls=client_cls) as protocol: + with cls(settings, client_cls=client_cls) as protocol: pass return protocol diff --git a/newrelic/core/config.py b/newrelic/core/config.py index ccd9a6132e..55b359174f 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -104,6 +104,7 @@ def create_settings(nested): class TopLevelSettings(Settings): _host = None + _otlp_host = None @property def host(self): @@ -115,6 +116,16 @@ def host(self): def host(self, value): self._host = value + @property + def otlp_host(self): + if self._otlp_host: + return self._otlp_host + return default_otlp_host(self.host) + + @otlp_host.setter + def otlp_host(self, value): + self._otlp_host = value + class AttributesSettings(Settings): pass @@ -560,6 +571,24 @@ def default_host(license_key): return host +def default_otlp_host(host): + HOST_MAP = { + "collector.newrelic.com": "otlp.nr-data.net", + "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", + "gov-collector.newrelic.com": "gov-otlp.nr-data.net", + "staging-collector.newrelic.com": "staging-otlp.nr-data.net", + "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", + "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", + "fake-collector.newrelic.com": "fake-otlp.nr-data.net", + } + otlp_host = HOST_MAP.get(host, None) + if not otlp_host: + default = HOST_MAP["collector.newrelic.com"] + _logger.warn("Unable to find corresponding OTLP host using default %s" % default) + otlp_host = default + return otlp_host + + _LOG_LEVEL = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, @@ -585,7 +614,9 @@ def default_host(license_key): _settings.ssl = _environ_as_bool("NEW_RELIC_SSL", True) _settings.host = os.environ.get("NEW_RELIC_HOST") +_settings.otlp_host = os.environ.get("NEW_RELIC_OTLP_HOST") _settings.port = int(os.environ.get("NEW_RELIC_PORT", "0")) +_settings.otlp_port = int(os.environ.get("NEW_RELIC_OTLP_PORT", "0")) _settings.agent_run_id = None _settings.entity_guid = None diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 2fdff4e434..eb5e75b839 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -36,6 +36,8 @@ _logger = logging.getLogger(__name__) +DIMENSIONAL_METRIC_DATA_TEMP = [] # TODO: REMOVE THIS + class Session(object): PROTOCOL = AgentProtocol diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py new file mode 100644 index 0000000000..6a44cb4e36 --- /dev/null +++ b/newrelic/core/otlp_utils.py @@ -0,0 +1,107 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module provides common utilities for interacting with OTLP protocol buffers.""" + +import logging + +_logger = logging.getLogger(__name__) + +try: + from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue + from newrelic.packages.opentelemetry_proto.logs_pb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, + ) + from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( + AggregationTemporality, + Metric, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, + ) + from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource + + AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + ValueAtQuantile = SummaryDataPoint.ValueAtQuantile + + otlp_encode = lambda payload: payload.SerializeToString() + OTLP_CONTENT_TYPE = "application/x-protobuf" + +except ImportError: + from newrelic.common.encoding_utils import json_encode + + def otlp_encode(*args, **kwargs): + _logger.warn( + "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." + ) + return json_encode(*args, **kwargs) + + Resource = dict + ValueAtQuantile = dict + AnyValue = dict + KeyValue = dict + NumberDataPoint = dict + SummaryDataPoint = dict + Sum = dict + Summary = dict + Metric = dict + MetricsData = dict + ScopeMetrics = dict + ResourceMetrics = dict + AGGREGATION_TEMPORALITY_DELTA = 1 + ResourceLogs = dict + ScopeLogs = dict + LogRecord = dict + OTLP_CONTENT_TYPE = "application/json" + + +def create_key_value(key, value): + if isinstance(value, bool): + return KeyValue(key=key, value=AnyValue(bool_value=value)) + elif isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + elif isinstance(value, float): + return KeyValue(key=key, value=AnyValue(double_value=value)) + elif isinstance(value, str): + return KeyValue(key=key, value=AnyValue(string_value=value)) + # Technically AnyValue accepts array, kvlist, and bytes however, since + # those are not valid custom attribute types according to our api spec, + # we will not bother to support them here either. + else: + _logger.warn("Unsupported attribute value type %s: %s." % (key, value)) + + +def create_key_values_from_iterable(iterable): + if isinstance(iterable, dict): + iterable = iterable.items() + + # The create_key_value list may return None if the value is an unsupported type + # so filter None values out before returning. + return list( + filter( + lambda i: i is not None, + (create_key_value(key, value) for key, value in iterable), + ) + ) + + +def create_resource(attributes=None): + attributes = attributes or {"instrumentation.provider": "nr_performance_monitoring"} + return Resource(attributes=create_key_values_from_iterable(attributes)) diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py index 5df69d71e9..547a0eeb69 100644 --- a/tests/agent_features/test_configuration.py +++ b/tests/agent_features/test_configuration.py @@ -577,6 +577,8 @@ def test_translate_deprecated_ignored_params_with_new_setting(): ("agent_run_id", None), ("entity_guid", None), ("distributed_tracing.exclude_newrelic_header", False), + ("otlp_host", "otlp.nr-data.net"), + ("otlp_port", 0), ), ) def test_default_values(name, expected_value): diff --git a/tests/agent_unittests/test_utilization_settings.py b/tests/agent_unittests/test_utilization_settings.py index 8af4bcbf1b..96cf47669c 100644 --- a/tests/agent_unittests/test_utilization_settings.py +++ b/tests/agent_unittests/test_utilization_settings.py @@ -118,6 +118,22 @@ def reset(wrapped, instance, args, kwargs): return reset +@reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) +def test_otlp_host_port_default(): + settings = global_settings() + assert settings.otlp_host == "otlp.nr-data.net" + assert settings.otlp_port == 0 + + +@reset_agent_config( + INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_OTLP_HOST": "custom-otlp.nr-data.net", "NEW_RELIC_OTLP_PORT": 443} +) +def test_otlp_port_override(): + settings = global_settings() + assert settings.otlp_host == "custom-otlp.nr-data.net" + assert settings.otlp_port == 443 + + @reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) def test_heroku_default(): settings = global_settings()