diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml new file mode 100644 index 0000000..dc521fb --- /dev/null +++ b/.github/workflows/check-vendor.yml @@ -0,0 +1,32 @@ +# This workflow will delete and regenerate the opentelemetry-exporter-otlp-proto-common code using scripts/vendor_otlp_proto_common.sh. +# If generating the code produces any changes from what is currently checked in, the workflow will fail and prompt the user to regenerate the code. +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Check OTLP Proto Common Vendored Code + +on: + push: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + pull_request: + branches: [ "main" ] + paths: + - "scripts/vendor_otlp_proto_common.sh" + - "src/snowflake/telemetry/_internal/opentelemetry/exporter/**" + - ".github/workflows/check-vendor.yml" + +jobs: + check-codegen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run vendor script + run: | + rm -rf src/snowflake/telemetry/_internal/opentelemetry/exporter/ + ./scripts/vendor_otlp_proto_common.sh + - name: Check for changes + run: | + git diff --exit-code || { echo "Code generation produced changes! Regenerate the code using ./scripts/vendor_otlp_proto_common.sh"; exit 1; } diff --git a/CHANGELOG.md b/CHANGELOG.md index 81b0510..ae5c320 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased * Upgrade OpenTelemetry Python dependencies to version 1.26.0 +* Vendored in adapter code from package opentelemetry-exporter-otlp-proto-common and replaced protobuf dependency with custom vanilla python serialization ## 0.5.0 (2024-07-23) diff --git a/anaconda/meta.yaml b/anaconda/meta.yaml index cd27e14..620a7ee 100644 --- a/anaconda/meta.yaml +++ b/anaconda/meta.yaml @@ -12,7 +12,6 @@ requirements: run: - python - opentelemetry-api ==1.26.0 - - opentelemetry-exporter-otlp-proto-common ==1.26.0 - opentelemetry-sdk ==1.26.0 about: diff --git a/benchmark/benchmark_serialize.py b/benchmark/benchmark_serialize.py new file mode 100644 index 0000000..e931ed6 --- /dev/null +++ b/benchmark/benchmark_serialize.py @@ -0,0 +1,86 @@ +import google_benchmark as benchmark + +from util import get_logs_data, get_metrics_data, get_traces_data, get_logs_data_4MB + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs as pb2_encode_logs +from opentelemetry.exporter.otlp.proto.common.metrics_encoder import encode_metrics as pb2_encode_metrics +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans as pb2_encode_spans + +""" +------------------------------------------------------------------------------ +Benchmark Time CPU Iterations +------------------------------------------------------------------------------ +test_bm_serialize_logs_data_4MB 730591536 ns 730562298 ns 1 +test_bm_pb2_serialize_logs_data_4MB 702522039 ns 702490893 ns 1 +test_bm_serialize_logs_data 100882 ns 100878 ns 6930 +test_bm_pb2_serialize_logs_data 97112 ns 97109 ns 7195 +test_bm_serialize_metrics_data 114938 ns 114934 ns 6096 +test_bm_pb2_serialize_metrics_data 161849 ns 161845 ns 4324 +test_bm_serialize_traces_data 123977 ns 123973 ns 5633 +test_bm_pb2_serialize_traces_data 131016 ns 131011 ns 5314 +""" + +def sanity_check(): + logs_data = get_logs_data() + metrics_data = get_metrics_data() + traces_data = get_traces_data() + + assert encode_logs(logs_data).SerializeToString() == pb2_encode_logs(logs_data).SerializeToString() + assert encode_metrics(metrics_data).SerializeToString() == pb2_encode_metrics(metrics_data).SerializeToString() + assert encode_spans(traces_data).SerializeToString() == pb2_encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data_4MB(state): + logs_data = get_logs_data_4MB() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_logs_data(state): + logs_data = get_logs_data() + while state: + pb2_encode_logs(logs_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_metrics_data(state): + metrics_data = get_metrics_data() + while state: + pb2_encode_metrics(metrics_data).SerializeToString() + +@benchmark.register +def test_bm_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + encode_spans(traces_data).SerializeToString() + +@benchmark.register +def test_bm_pb2_serialize_traces_data(state): + traces_data = get_traces_data() + while state: + pb2_encode_spans(traces_data).SerializeToString() + +if __name__ == "__main__": + sanity_check() + benchmark.main() \ No newline at end of file diff --git a/benchmark/util.py b/benchmark/util.py new file mode 100644 index 0000000..e73bc56 --- /dev/null +++ b/benchmark/util.py @@ -0,0 +1,339 @@ +from typing import Sequence + +from snowflake.telemetry.test.metrictestutil import _generate_gauge, _generate_sum + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.util.instrumentation import InstrumentationScope + +from opentelemetry._logs import SeverityNumber +from opentelemetry.sdk._logs import LogData, LogLimits, LogRecord + +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, + Buckets, + ExponentialHistogram, + Histogram, + ExponentialHistogramDataPoint, + HistogramDataPoint, + Metric, + MetricsData, + ResourceMetrics, + ScopeMetrics, +) + +from opentelemetry.sdk.trace import Event, SpanContext, _Span +from opentelemetry.trace import SpanKind, Link, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + + + +def get_logs_data() -> Sequence[LogData]: + log1 = LogData( + log_record=LogRecord( + timestamp=1644650195189786880, + observed_timestamp=1644660000000000000, + trace_id=89564621134313219400156819398935297684, + span_id=1312458408527513268, + trace_flags=TraceFlags(0x01), + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Do not go gentle into that good night. Rage, rage against the dying of the light", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=InstrumentationScope( + "first_name", "first_version" + ), + ) + + log2 = LogData( + log_record=LogRecord( + timestamp=1644650249738562048, + observed_timestamp=1644660000000000000, + trace_id=0, + span_id=0, + trace_flags=TraceFlags.DEFAULT, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Cooper, this is no time for caution!", + resource=Resource({"second_resource": "CASE"}), + attributes={}, + ), + instrumentation_scope=InstrumentationScope( + "second_name", "second_version" + ), + ) + + log3 = LogData( + log_record=LogRecord( + timestamp=1644650427658989056, + observed_timestamp=1644660000000000000, + trace_id=271615924622795969659406376515024083555, + span_id=4242561578944770265, + trace_flags=TraceFlags(0x01), + severity_text="DEBUG", + severity_number=SeverityNumber.DEBUG, + body="To our galaxy", + resource=Resource({"second_resource": "CASE"}), + attributes={"a": 1, "b": "c"}, + ), + instrumentation_scope=None, + ) + + log4 = LogData( + log_record=LogRecord( + timestamp=1644650584292683008, + observed_timestamp=1644660000000000000, + trace_id=212592107417388365804938480559624925555, + span_id=6077757853989569223, + trace_flags=TraceFlags(0x01), + severity_text="INFO", + severity_number=SeverityNumber.INFO, + body="Love is the one thing that transcends time and space", + resource=Resource( + {"first_resource": "value"}, + "resource_schema_url", + ), + attributes={"filename": "model.py", "func_name": "run_method"}, + ), + instrumentation_scope=InstrumentationScope( + "another_name", "another_version" + ), + ) + + return [log1, log2, log3, log4] + +def get_logs_data_4MB() -> Sequence[LogData]: + out = [] + for _ in range(8000): + out.extend(get_logs_data()) + return out + +HISTOGRAM = Metric( + name="histogram", + description="foo", + unit="s", + data=Histogram( + data_points=[ + HistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=1641946016139533244, + time_unix_nano=1641946016139533244, + count=5, + sum=67, + bucket_counts=[1, 4], + explicit_bounds=[10.0, 20.0], + min=8, + max=18, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) + +EXPONENTIAL_HISTOGRAM = Metric( + name="exponential_histogram", + description="description", + unit="unit", + data=ExponentialHistogram( + data_points=[ + ExponentialHistogramDataPoint( + attributes={"a": 1, "b": True}, + start_time_unix_nano=0, + time_unix_nano=1, + count=2, + sum=3, + scale=4, + zero_count=5, + positive=Buckets(offset=6, bucket_counts=[7, 8]), + negative=Buckets(offset=9, bucket_counts=[10, 11]), + flags=12, + min=13.0, + max=14.0, + ) + ], + aggregation_temporality=AggregationTemporality.DELTA, + ), +) +def get_metrics_data() -> MetricsData: + + metrics = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource( + attributes={"a": 1, "b": False}, + schema_url="resource_schema_url", + ), + scope_metrics=[ + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM, HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="second_name", + version="second_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="third_name", + version="third_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_int", 33)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_sum("sum_double", 2.98)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_int", 9000)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[_generate_gauge("gauge_double", 52.028)], + schema_url="instrumentation_scope_schema_url", + ), + ScopeMetrics( + scope=InstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=[EXPONENTIAL_HISTOGRAM], + schema_url="instrumentation_scope_schema_url", + ) + ], + schema_url="resource_schema_url", + ) + ] + ) + + return metrics + +def get_traces_data() -> Sequence[_Span]: + trace_id = 0x3E0C63257DE34C926F9EFCD03927272E + + base_time = 683647322 * 10**9 # in ns + start_times = ( + base_time, + base_time + 150 * 10**6, + base_time + 300 * 10**6, + base_time + 400 * 10**6, + ) + end_times = ( + start_times[0] + (50 * 10**6), + start_times[1] + (100 * 10**6), + start_times[2] + (200 * 10**6), + start_times[3] + (300 * 10**6), + ) + + parent_span_context = SpanContext( + trace_id, 0x1111111111111111, is_remote=True + ) + + other_context = SpanContext( + trace_id, 0x2222222222222222, is_remote=False + ) + + span1 = _Span( + name="test-span-1", + context=SpanContext( + trace_id, + 0x34BF92DEEFC58C92, + is_remote=False, + trace_flags=TraceFlags(TraceFlags.SAMPLED), + ), + parent=parent_span_context, + events=( + Event( + name="event0", + timestamp=base_time + 50 * 10**6, + attributes={ + "annotation_bool": True, + "annotation_string": "annotation_test", + "key_float": 0.3, + }, + ), + ), + links=( + Link(context=other_context, attributes={"key_bool": True}), + ), + resource=Resource({}, "resource_schema_url"), + ) + span1.start(start_time=start_times[0]) + span1.set_attribute("key_bool", False) + span1.set_attribute("key_string", "hello_world") + span1.set_attribute("key_float", 111.22) + span1.set_status(Status(StatusCode.ERROR, "Example description")) + span1.end(end_time=end_times[0]) + + span2 = _Span( + name="test-span-2", + context=parent_span_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span2.start(start_time=start_times[1]) + span2.end(end_time=end_times[1]) + + span3 = _Span( + name="test-span-3", + context=other_context, + parent=None, + resource=Resource(attributes={"key_resource": "some_resource"}), + ) + span3.start(start_time=start_times[2]) + span3.set_attribute("key_string", "hello_world") + span3.end(end_time=end_times[2]) + + span4 = _Span( + name="test-span-4", + context=other_context, + parent=None, + resource=Resource({}, "resource_schema_url"), + instrumentation_scope=InstrumentationScope( + name="name", version="version" + ), + ) + span4.start(start_time=start_times[3]) + span4.end(end_time=end_times[3]) + + return [span1, span2, span3, span4] \ No newline at end of file diff --git a/scripts/vendor_otlp_proto_common.sh b/scripts/vendor_otlp_proto_common.sh new file mode 100755 index 0000000..eae16a2 --- /dev/null +++ b/scripts/vendor_otlp_proto_common.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Vendor in the python code in +# https://github.com/open-telemetry/opentelemetry-python/tree/main/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter/otlp/proto/common +# +# To use, update REPO_BRANCH_OR_COMMIT variable below to a commit hash or +# tag in opentelemtry-python repo that you want to build off of. Then, just run +# this script to update the proto files. Commit the changes as well as any +# fixes needed in the OTLP exporter. + +# Pinned commit/branch/tag for the current version used in opentelemetry-proto python package. +REPO_BRANCH_OR_COMMIT="v1.26.0" + +set -e + +REPO_DIR=${REPO_DIR:-"/tmp/opentelemetry-python"} +# root of opentelemetry-python repo +repo_root="$(git rev-parse --show-toplevel)" + +# Clone the proto repo if it doesn't exist +if [ ! -d "$REPO_DIR" ]; then + git clone https://github.com/open-telemetry/opentelemetry-python.git $REPO_DIR +fi + +# Pull in changes and switch to requested branch +( + cd $REPO_DIR + git fetch --all + git checkout $REPO_BRANCH_OR_COMMIT + # pull if REPO_BRANCH_OR_COMMIT is not a detached head + git symbolic-ref -q HEAD && git pull --ff-only || true +) + +cd $repo_root/src/snowflake/telemetry/_internal + +# Copy the entire file tree from exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/ +# to src/snowflake/telemetry/_internal/opentelemetry/ +cp -r $REPO_DIR/exporter/opentelemetry-exporter-otlp-proto-common/src/opentelemetry/exporter opentelemetry/ + +# If MacOS need '' after -i +# Detect the OS (macOS or Linux) +if [[ "$OSTYPE" == "darwin"* ]]; then + SED_CMD="sed -i ''" +else + SED_CMD="sed -i" +fi + +# Replace all the imports strings in the copied python files +# opentelemetry.exporter to snowflake.telemetry._internal.opentelemetry.exporter +# opentelemetry.proto.*_pb2 to snowflake.telemetry._internal.opentelemetry.proto.*_marshaler + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry.exporter/snowflake.telemetry._internal.opentelemetry.exporter/g' {} + +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD 's/opentelemetry\.proto\(.*\)_pb2/snowflake.telemetry._internal.opentelemetry.proto\1_marshaler/g' {} + + + +# Add a notice to the top of every file in compliance with Apache 2.0 to indicate that the file has been modified +# https://www.apache.org/licenses/LICENSE-2.0 +find opentelemetry/exporter -type f -name "*.py" -exec $SED_CMD '14s|^|#\n# This file has been modified from the original source code at\n#\n# https://github.com/open-telemetry/opentelemetry-python/tree/'"$REPO_BRANCH_OR_COMMIT"'\n#\n# by Snowflake Inc.\n|' {} + diff --git a/setup.py b/setup.py index 06b813c..5e2f0e7 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ long_description=LONG_DESCRIPTION, install_requires=[ "opentelemetry-api == 1.26.0", - "opentelemetry-exporter-otlp-proto-common == 1.26.0", "opentelemetry-sdk == 1.26.0", ], packages=find_namespace_packages( diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py index 3214e3e..9361c20 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/logs/__init__.py @@ -21,10 +21,10 @@ import opentelemetry.sdk.util.instrumentation as otel_instrumentation import opentelemetry.sdk._logs._internal as _logs_internal -from opentelemetry.exporter.otlp.proto.common._log_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import ( encode_logs, ) -from opentelemetry.proto.logs.v1.logs_pb2 import LogsData +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogsData from opentelemetry.sdk.resources import Resource from opentelemetry.sdk._logs import export from opentelemetry.sdk import _logs diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py index 46291d1..91ff137 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/metrics/__init__.py @@ -17,10 +17,10 @@ from typing import Dict import opentelemetry -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) -from opentelemetry.proto.metrics.v1.metrics_pb2 import MetricsData as PB2MetricsData +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1.metrics_marshaler import MetricsData as PB2MetricsData from opentelemetry.sdk.metrics.export import ( AggregationTemporality, MetricExportResult, diff --git a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py index 7c877aa..41aa617 100644 --- a/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py +++ b/src/snowflake/telemetry/_internal/exporter/otlp/proto/traces/__init__.py @@ -16,10 +16,10 @@ import abc import typing -from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import ( encode_spans, ) -from opentelemetry.proto.trace.v1.trace_pb2 import TracesData +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import TracesData from opentelemetry.sdk.trace import ReadableSpan from opentelemetry.sdk.trace.export import ( SpanExportResult, diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py new file mode 100644 index 0000000..c1250a0 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/__init__.py @@ -0,0 +1,24 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ + +__all__ = ["__version__"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py new file mode 100644 index 0000000..530528f --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/__init__.py @@ -0,0 +1,182 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +import logging +from collections.abc import Sequence +from itertools import count +from typing import ( + Any, + Mapping, + Optional, + List, + Callable, + TypeVar, + Dict, + Iterator, +) + +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + InstrumentationScope as PB2InstrumentationScope, +) +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import AnyValue as PB2AnyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import KeyValue as PB2KeyValue +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + KeyValueList as PB2KeyValueList, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import ( + ArrayValue as PB2ArrayValue, +) +from opentelemetry.sdk.trace import Resource +from opentelemetry.util.types import Attributes + +_logger = logging.getLogger(__name__) + +_TypingResourceT = TypeVar("_TypingResourceT") +_ResourceDataT = TypeVar("_ResourceDataT") + + +def _encode_instrumentation_scope( + instrumentation_scope: InstrumentationScope, +) -> PB2InstrumentationScope: + if instrumentation_scope is None: + return PB2InstrumentationScope() + return PB2InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + + +def _encode_resource(resource: Resource) -> PB2Resource: + return PB2Resource(attributes=_encode_attributes(resource.attributes)) + + +def _encode_value(value: Any) -> PB2AnyValue: + if isinstance(value, bool): + return PB2AnyValue(bool_value=value) + if isinstance(value, str): + return PB2AnyValue(string_value=value) + if isinstance(value, int): + return PB2AnyValue(int_value=value) + if isinstance(value, float): + return PB2AnyValue(double_value=value) + if isinstance(value, Sequence): + return PB2AnyValue( + array_value=PB2ArrayValue(values=[_encode_value(v) for v in value]) + ) + elif isinstance(value, Mapping): + return PB2AnyValue( + kvlist_value=PB2KeyValueList( + values=[_encode_key_value(str(k), v) for k, v in value.items()] + ) + ) + raise Exception(f"Invalid type {type(value)} of value {value}") + + +def _encode_key_value(key: str, value: Any) -> PB2KeyValue: + return PB2KeyValue(key=key, value=_encode_value(value)) + + +def _encode_span_id(span_id: int) -> bytes: + return span_id.to_bytes(length=8, byteorder="big", signed=False) + + +def _encode_trace_id(trace_id: int) -> bytes: + return trace_id.to_bytes(length=16, byteorder="big", signed=False) + + +def _encode_attributes( + attributes: Attributes, +) -> Optional[List[PB2KeyValue]]: + if attributes: + pb2_attributes = [] + for key, value in attributes.items(): + # pylint: disable=broad-exception-caught + try: + pb2_attributes.append(_encode_key_value(key, value)) + except Exception as error: + _logger.exception("Failed to encode key %s: %s", key, error) + else: + pb2_attributes = None + return pb2_attributes + + +def _get_resource_data( + sdk_resource_scope_data: Dict[Resource, _ResourceDataT], + resource_class: Callable[..., _TypingResourceT], + name: str, +) -> List[_TypingResourceT]: + resource_data = [] + + for ( + sdk_resource, + scope_data, + ) in sdk_resource_scope_data.items(): + collector_resource = PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ) + resource_data.append( + resource_class( + **{ + "resource": collector_resource, + "scope_{}".format(name): scope_data.values(), + } + ) + ) + return resource_data + + +def _create_exp_backoff_generator(max_value: int = 0) -> Iterator[int]: + """ + Generates an infinite sequence of exponential backoff values. The sequence starts + from 1 (2^0) and doubles each time (2^1, 2^2, 2^3, ...). If a max_value is specified + and non-zero, the generated values will not exceed this maximum, capping at max_value + instead of growing indefinitely. + + Parameters: + - max_value (int, optional): The maximum value to yield. If 0 or not provided, the + sequence grows without bound. + + Returns: + Iterator[int]: An iterator that yields the exponential backoff values, either uncapped or + capped at max_value. + + Example: + ``` + gen = _create_exp_backoff_generator(max_value=10) + for _ in range(5): + print(next(gen)) + ``` + This will print: + 1 + 2 + 4 + 8 + 10 + + Note: this functionality used to be handled by the 'backoff' package. + """ + for i in count(0): + out = 2**i + yield min(out, max_value) if max_value else out diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py new file mode 100644 index 0000000..2f71a17 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/_log_encoder/__init__.py @@ -0,0 +1,101 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +from collections import defaultdict +from typing import Sequence, List + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, + _encode_value, + _encode_attributes, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.logs.v1.logs_service_marshaler import ( + ExportLogsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import ( + ScopeLogs, + ResourceLogs, +) +from snowflake.telemetry._internal.opentelemetry.proto.logs.v1.logs_marshaler import LogRecord as PB2LogRecord + +from opentelemetry.sdk._logs import LogData + + +def encode_logs(batch: Sequence[LogData]) -> ExportLogsServiceRequest: + return ExportLogsServiceRequest(resource_logs=_encode_resource_logs(batch)) + + +def _encode_log(log_data: LogData) -> PB2LogRecord: + span_id = ( + None + if log_data.log_record.span_id == 0 + else _encode_span_id(log_data.log_record.span_id) + ) + trace_id = ( + None + if log_data.log_record.trace_id == 0 + else _encode_trace_id(log_data.log_record.trace_id) + ) + return PB2LogRecord( + time_unix_nano=log_data.log_record.timestamp, + observed_time_unix_nano=log_data.log_record.observed_timestamp, + span_id=span_id, + trace_id=trace_id, + flags=int(log_data.log_record.trace_flags), + body=_encode_value(log_data.log_record.body), + severity_text=log_data.log_record.severity_text, + attributes=_encode_attributes(log_data.log_record.attributes), + dropped_attributes_count=log_data.log_record.dropped_attributes, + severity_number=log_data.log_record.severity_number.value, + ) + + +def _encode_resource_logs(batch: Sequence[LogData]) -> List[ResourceLogs]: + sdk_resource_logs = defaultdict(lambda: defaultdict(list)) + + for sdk_log in batch: + sdk_resource = sdk_log.log_record.resource + sdk_instrumentation = sdk_log.instrumentation_scope or None + pb2_log = _encode_log(sdk_log) + + sdk_resource_logs[sdk_resource][sdk_instrumentation].append(pb2_log) + + pb2_resource_logs = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_logs.items(): + scope_logs = [] + for sdk_instrumentation, pb2_logs in sdk_instrumentations.items(): + scope_logs.append( + ScopeLogs( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + log_records=pb2_logs, + ) + ) + pb2_resource_logs.append( + ResourceLogs( + resource=_encode_resource(sdk_resource), + scope_logs=scope_logs, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_logs diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py new file mode 100644 index 0000000..9d729cd --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/metrics_encoder/__init__.py @@ -0,0 +1,344 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. +import logging + +from opentelemetry.sdk.metrics.export import ( + MetricExporter, +) +from opentelemetry.sdk.metrics.view import Aggregation +from os import environ +from opentelemetry.sdk.metrics import ( + Counter, + Histogram, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, +) +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, +) +from opentelemetry.sdk.metrics.export import ( + AggregationTemporality, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.metrics.v1.metrics_service_marshaler import ( + ExportMetricsServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.common.v1.common_marshaler import InstrumentationScope +from snowflake.telemetry._internal.opentelemetry.proto.metrics.v1 import metrics_marshaler as pb2 +from opentelemetry.sdk.metrics.export import ( + MetricsData, + Gauge, + Histogram as HistogramType, + Sum, + ExponentialHistogram as ExponentialHistogramType, +) +from typing import Dict +from snowflake.telemetry._internal.opentelemetry.proto.resource.v1.resource_marshaler import ( + Resource as PB2Resource, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, +) +from opentelemetry.sdk.metrics.view import ( + ExponentialBucketHistogramAggregation, + ExplicitBucketHistogramAggregation, +) + +_logger = logging.getLogger(__name__) + + +class OTLPMetricExporterMixin: + def _common_configuration( + self, + preferred_temporality: Dict[type, AggregationTemporality] = None, + preferred_aggregation: Dict[type, Aggregation] = None, + ) -> None: + + MetricExporter.__init__( + self, + preferred_temporality=self._get_temporality(preferred_temporality), + preferred_aggregation=self._get_aggregation(preferred_aggregation), + ) + + def _get_temporality( + self, preferred_temporality: Dict[type, AggregationTemporality] + ) -> Dict[type, AggregationTemporality]: + + otel_exporter_otlp_metrics_temporality_preference = ( + environ.get( + OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, + "CUMULATIVE", + ) + .upper() + .strip() + ) + + if otel_exporter_otlp_metrics_temporality_preference == "DELTA": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.DELTA, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + elif otel_exporter_otlp_metrics_temporality_preference == "LOWMEMORY": + instrument_class_temporality = { + Counter: AggregationTemporality.DELTA, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.DELTA, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + else: + if otel_exporter_otlp_metrics_temporality_preference != ( + "CUMULATIVE" + ): + _logger.warning( + "Unrecognized OTEL_EXPORTER_METRICS_TEMPORALITY_PREFERENCE" + " value found: " + f"{otel_exporter_otlp_metrics_temporality_preference}, " + "using CUMULATIVE" + ) + instrument_class_temporality = { + Counter: AggregationTemporality.CUMULATIVE, + UpDownCounter: AggregationTemporality.CUMULATIVE, + Histogram: AggregationTemporality.CUMULATIVE, + ObservableCounter: AggregationTemporality.CUMULATIVE, + ObservableUpDownCounter: AggregationTemporality.CUMULATIVE, + ObservableGauge: AggregationTemporality.CUMULATIVE, + } + + instrument_class_temporality.update(preferred_temporality or {}) + + return instrument_class_temporality + + def _get_aggregation( + self, + preferred_aggregation: Dict[type, Aggregation], + ) -> Dict[type, Aggregation]: + + otel_exporter_otlp_metrics_default_histogram_aggregation = environ.get( + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + "explicit_bucket_histogram", + ) + + if otel_exporter_otlp_metrics_default_histogram_aggregation == ( + "base2_exponential_bucket_histogram" + ): + + instrument_class_aggregation = { + Histogram: ExponentialBucketHistogramAggregation(), + } + + else: + + if otel_exporter_otlp_metrics_default_histogram_aggregation != ( + "explicit_bucket_histogram" + ): + + _logger.warning( + ( + "Invalid value for %s: %s, using explicit bucket " + "histogram aggregation" + ), + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, + otel_exporter_otlp_metrics_default_histogram_aggregation, + ) + + instrument_class_aggregation = { + Histogram: ExplicitBucketHistogramAggregation(), + } + + instrument_class_aggregation.update(preferred_aggregation or {}) + + return instrument_class_aggregation + + +def encode_metrics(data: MetricsData) -> ExportMetricsServiceRequest: + resource_metrics_dict = {} + + for resource_metrics in data.resource_metrics: + + resource = resource_metrics.resource + + # It is safe to assume that each entry in data.resource_metrics is + # associated with an unique resource. + scope_metrics_dict = {} + + resource_metrics_dict[resource] = scope_metrics_dict + + for scope_metrics in resource_metrics.scope_metrics: + + instrumentation_scope = scope_metrics.scope + + # The SDK groups metrics in instrumentation scopes already so + # there is no need to check for existing instrumentation scopes + # here. + pb2_scope_metrics = pb2.ScopeMetrics( + scope=InstrumentationScope( + name=instrumentation_scope.name, + version=instrumentation_scope.version, + ) + ) + + scope_metrics_dict[instrumentation_scope] = pb2_scope_metrics + + for metric in scope_metrics.metrics: + pb2_metric = pb2.Metric( + name=metric.name, + description=metric.description, + unit=metric.unit, + ) + + if isinstance(metric.data, Gauge): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + pb2_metric.gauge.data_points.append(pt) + + elif isinstance(metric.data, HistogramType): + for data_point in metric.data.data_points: + pt = pb2.HistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + bucket_counts=data_point.bucket_counts, + explicit_bounds=data_point.explicit_bounds, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.histogram.data_points.append(pt) + + elif isinstance(metric.data, Sum): + for data_point in metric.data.data_points: + pt = pb2.NumberDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + time_unix_nano=data_point.time_unix_nano, + ) + if isinstance(data_point.value, int): + pt.as_int = data_point.value + else: + pt.as_double = data_point.value + # note that because sum is a message type, the + # fields must be set individually rather than + # instantiating a pb2.Sum and setting it once + pb2_metric.sum.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.sum.is_monotonic = metric.data.is_monotonic + pb2_metric.sum.data_points.append(pt) + + elif isinstance(metric.data, ExponentialHistogramType): + for data_point in metric.data.data_points: + + if data_point.positive.bucket_counts: + positive = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.positive.offset, + bucket_counts=data_point.positive.bucket_counts, + ) + else: + positive = None + + if data_point.negative.bucket_counts: + negative = pb2.ExponentialHistogramDataPoint.Buckets( + offset=data_point.negative.offset, + bucket_counts=data_point.negative.bucket_counts, + ) + else: + negative = None + + pt = pb2.ExponentialHistogramDataPoint( + attributes=_encode_attributes( + data_point.attributes + ), + time_unix_nano=data_point.time_unix_nano, + start_time_unix_nano=( + data_point.start_time_unix_nano + ), + count=data_point.count, + sum=data_point.sum, + scale=data_point.scale, + zero_count=data_point.zero_count, + positive=positive, + negative=negative, + flags=data_point.flags, + max=data_point.max, + min=data_point.min, + ) + pb2_metric.exponential_histogram.aggregation_temporality = ( + metric.data.aggregation_temporality + ) + pb2_metric.exponential_histogram.data_points.append(pt) + + else: + _logger.warning( + "unsupported data type %s", + metric.data.__class__.__name__, + ) + continue + + pb2_scope_metrics.metrics.append(pb2_metric) + + resource_data = [] + for ( + sdk_resource, + scope_data, + ) in resource_metrics_dict.items(): + resource_data.append( + pb2.ResourceMetrics( + resource=PB2Resource( + attributes=_encode_attributes(sdk_resource.attributes) + ), + scope_metrics=scope_data.values(), + schema_url=sdk_resource.schema_url, + ) + ) + resource_metrics = resource_data + return ExportMetricsServiceRequest(resource_metrics=resource_metrics) diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py new file mode 100644 index 0000000..e269641 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_internal/trace_encoder/__init__.py @@ -0,0 +1,195 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +import logging +from collections import defaultdict +from typing import List, Optional, Sequence + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal import ( + _encode_attributes, + _encode_instrumentation_scope, + _encode_resource, + _encode_span_id, + _encode_trace_id, +) +from snowflake.telemetry._internal.opentelemetry.proto.collector.trace.v1.trace_service_marshaler import ( + ExportTraceServiceRequest as PB2ExportTraceServiceRequest, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ( + ResourceSpans as PB2ResourceSpans, +) +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import ScopeSpans as PB2ScopeSpans +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Span as PB2SPan +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import SpanFlags as PB2SpanFlags +from snowflake.telemetry._internal.opentelemetry.proto.trace.v1.trace_marshaler import Status as PB2Status +from opentelemetry.sdk.trace import Event, ReadableSpan +from opentelemetry.trace import Link, SpanKind +from opentelemetry.trace.span import SpanContext, Status, TraceState + +# pylint: disable=E1101 +_SPAN_KIND_MAP = { + SpanKind.INTERNAL: PB2SPan.SpanKind.SPAN_KIND_INTERNAL, + SpanKind.SERVER: PB2SPan.SpanKind.SPAN_KIND_SERVER, + SpanKind.CLIENT: PB2SPan.SpanKind.SPAN_KIND_CLIENT, + SpanKind.PRODUCER: PB2SPan.SpanKind.SPAN_KIND_PRODUCER, + SpanKind.CONSUMER: PB2SPan.SpanKind.SPAN_KIND_CONSUMER, +} + +_logger = logging.getLogger(__name__) + + +def encode_spans( + sdk_spans: Sequence[ReadableSpan], +) -> PB2ExportTraceServiceRequest: + return PB2ExportTraceServiceRequest( + resource_spans=_encode_resource_spans(sdk_spans) + ) + + +def _encode_resource_spans( + sdk_spans: Sequence[ReadableSpan], +) -> List[PB2ResourceSpans]: + # We need to inspect the spans and group + structure them as: + # + # Resource + # Instrumentation Library + # Spans + # + # First loop organizes the SDK spans in this structure. Protobuf messages + # are not hashable so we stick with SDK data in this phase. + # + # Second loop encodes the data into Protobuf format. + # + sdk_resource_spans = defaultdict(lambda: defaultdict(list)) + + for sdk_span in sdk_spans: + sdk_resource = sdk_span.resource + sdk_instrumentation = sdk_span.instrumentation_scope or None + pb2_span = _encode_span(sdk_span) + + sdk_resource_spans[sdk_resource][sdk_instrumentation].append(pb2_span) + + pb2_resource_spans = [] + + for sdk_resource, sdk_instrumentations in sdk_resource_spans.items(): + scope_spans = [] + for sdk_instrumentation, pb2_spans in sdk_instrumentations.items(): + scope_spans.append( + PB2ScopeSpans( + scope=(_encode_instrumentation_scope(sdk_instrumentation)), + spans=pb2_spans, + ) + ) + pb2_resource_spans.append( + PB2ResourceSpans( + resource=_encode_resource(sdk_resource), + scope_spans=scope_spans, + schema_url=sdk_resource.schema_url, + ) + ) + + return pb2_resource_spans + + +def _span_flags(parent_span_context: Optional[SpanContext]) -> int: + flags = PB2SpanFlags.SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK + if parent_span_context and parent_span_context.is_remote: + flags |= PB2SpanFlags.SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK + return flags + + +def _encode_span(sdk_span: ReadableSpan) -> PB2SPan: + span_context = sdk_span.get_span_context() + return PB2SPan( + trace_id=_encode_trace_id(span_context.trace_id), + span_id=_encode_span_id(span_context.span_id), + trace_state=_encode_trace_state(span_context.trace_state), + parent_span_id=_encode_parent_id(sdk_span.parent), + name=sdk_span.name, + kind=_SPAN_KIND_MAP[sdk_span.kind], + start_time_unix_nano=sdk_span.start_time, + end_time_unix_nano=sdk_span.end_time, + attributes=_encode_attributes(sdk_span.attributes), + events=_encode_events(sdk_span.events), + links=_encode_links(sdk_span.links), + status=_encode_status(sdk_span.status), + dropped_attributes_count=sdk_span.dropped_attributes, + dropped_events_count=sdk_span.dropped_events, + dropped_links_count=sdk_span.dropped_links, + flags=_span_flags(sdk_span.parent), + ) + + +def _encode_events( + events: Sequence[Event], +) -> Optional[List[PB2SPan.Event]]: + pb2_events = None + if events: + pb2_events = [] + for event in events: + encoded_event = PB2SPan.Event( + name=event.name, + time_unix_nano=event.timestamp, + attributes=_encode_attributes(event.attributes), + dropped_attributes_count=event.dropped_attributes, + ) + pb2_events.append(encoded_event) + return pb2_events + + +def _encode_links(links: Sequence[Link]) -> Sequence[PB2SPan.Link]: + pb2_links = None + if links: + pb2_links = [] + for link in links: + encoded_link = PB2SPan.Link( + trace_id=_encode_trace_id(link.context.trace_id), + span_id=_encode_span_id(link.context.span_id), + attributes=_encode_attributes(link.attributes), + dropped_attributes_count=link.attributes.dropped, + flags=_span_flags(link.context), + ) + pb2_links.append(encoded_link) + return pb2_links + + +def _encode_status(status: Status) -> Optional[PB2Status]: + pb2_status = None + if status is not None: + pb2_status = PB2Status( + code=status.status_code.value, + message=status.description, + ) + return pb2_status + + +def _encode_trace_state(trace_state: TraceState) -> Optional[str]: + pb2_trace_state = None + if trace_state is not None: + pb2_trace_state = ",".join( + [f"{key}={value}" for key, value in (trace_state.items())] + ) + return pb2_trace_state + + +def _encode_parent_id(context: Optional[SpanContext]) -> Optional[bytes]: + if context: + return _encode_span_id(context.span_id) + return None diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py new file mode 100644 index 0000000..481a853 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/_log_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal._log_encoder import ( + encode_logs, +) + +__all__ = ["encode_logs"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py new file mode 100644 index 0000000..4d82926 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/metrics_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder import ( + encode_metrics, +) + +__all__ = ["encode_metrics"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py new file mode 100644 index 0000000..bc37212 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/trace_encoder.py @@ -0,0 +1,26 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( + encode_spans, +) + +__all__ = ["encode_spans"] diff --git a/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py new file mode 100644 index 0000000..8ddc9b6 --- /dev/null +++ b/src/snowflake/telemetry/_internal/opentelemetry/exporter/otlp/proto/common/version.py @@ -0,0 +1,21 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file has been modified from the original source code at +# +# https://github.com/open-telemetry/opentelemetry-python/tree/v1.26.0 +# +# by Snowflake Inc. + +__version__ = "1.26.0" diff --git a/tests/snowflake-telemetry-test-utils/setup.py b/tests/snowflake-telemetry-test-utils/setup.py index 0511b8e..6254955 100644 --- a/tests/snowflake-telemetry-test-utils/setup.py +++ b/tests/snowflake-telemetry-test-utils/setup.py @@ -15,6 +15,7 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, install_requires=[ + "opentelemetry-exporter-otlp-proto-common == 1.26.0", "pytest >= 7.0.0", "snowflake-telemetry-python == 0.6.0.dev", "Jinja2 == 3.1.4", @@ -22,6 +23,7 @@ "black >= 24.1.0", "isort >= 5.12.0", "hypothesis >= 6.0.0", + "google-benchmark", ], packages=find_namespace_packages( where='src' diff --git a/tests/test_log_encoder.py b/tests/test_log_encoder.py index a4cfb95..dcf0e4e 100644 --- a/tests/test_log_encoder.py +++ b/tests/test_log_encoder.py @@ -22,7 +22,7 @@ _encode_trace_id, _encode_value, ) -from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( ExportLogsServiceRequest, ) @@ -55,7 +55,7 @@ class TestOTLPLogEncoder(unittest.TestCase): def test_encode(self): sdk_logs, expected_encoding = self.get_test_logs() - self.assertEqual(encode_logs(sdk_logs), expected_encoding) + self.assertEqual(encode_logs(sdk_logs).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_log_exporter(self): sdk_logs, expected_encoding = self.get_test_logs() @@ -69,11 +69,13 @@ def test_proto_log_exporter(self): def test_dropped_attributes_count(self): sdk_logs = self._get_test_logs_dropped_attributes() - encoded_logs = encode_logs(sdk_logs) + encoded_logs = bytes(encode_logs(sdk_logs)) + decoded_logs = PB2LogsData() + decoded_logs.ParseFromString(encoded_logs) self.assertTrue(hasattr(sdk_logs[0].log_record, "dropped_attributes")) self.assertEqual( # pylint:disable=no-member - encoded_logs.resource_logs[0] + decoded_logs.resource_logs[0] .scope_logs[0] .log_records[0] .dropped_attributes_count, diff --git a/tests/test_metrics_encoder.py b/tests/test_metrics_encoder.py index 76464a6..7a473ed 100644 --- a/tests/test_metrics_encoder.py +++ b/tests/test_metrics_encoder.py @@ -15,7 +15,7 @@ # pylint: disable=protected-access import unittest -from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( @@ -159,8 +159,8 @@ def test_encode_sum_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -245,8 +245,8 @@ def test_encode_sum_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -328,8 +328,8 @@ def test_encode_gauge_int(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -411,8 +411,8 @@ def test_encode_gauge_double(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -502,8 +502,8 @@ def test_encode_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -730,8 +730,8 @@ def test_encode_multiple_scope_histogram(self): ) ] ) - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() @@ -856,8 +856,8 @@ def test_encode_exponential_histogram(self): ] ) # pylint: disable=protected-access - actual = encode_metrics(metrics_data) - self.assertEqual(expected, actual) + actual = encode_metrics(metrics_data).SerializeToString() + self.assertEqual(expected.SerializeToString(), actual) self.metric_writer.clear() self.exporter.export(metrics_data) protos = self.metric_writer.get_finished_protos() diff --git a/tests/test_trace_encoder.py b/tests/test_trace_encoder.py index 322a521..d8b326e 100644 --- a/tests/test_trace_encoder.py +++ b/tests/test_trace_encoder.py @@ -25,7 +25,7 @@ _SPAN_KIND_MAP, _encode_status, ) -from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( ExportTraceServiceRequest as PB2ExportTraceServiceRequest, ) @@ -67,7 +67,7 @@ class TestOTLPTraceEncoder(unittest.TestCase): def test_encode_spans(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() - self.assertEqual(encode_spans(otel_spans), expected_encoding) + self.assertEqual(encode_spans(otel_spans).SerializeToString(), expected_encoding.SerializeToString()) def test_proto_span_exporter(self): otel_spans, expected_encoding = self.get_exhaustive_test_spans() diff --git a/tests/test_vendored_exporter_version.py b/tests/test_vendored_exporter_version.py new file mode 100644 index 0000000..d6509b1 --- /dev/null +++ b/tests/test_vendored_exporter_version.py @@ -0,0 +1,11 @@ +import unittest + +from snowflake.telemetry._internal.opentelemetry.exporter.otlp.proto.common.version import __version__ as vendored_version + +from opentelemetry.sdk.version import __version__ as sdk_version +from opentelemetry.exporter.otlp.proto.common.version import __version__ as exporter_version + +class TestVendoredExporterVersion(unittest.TestCase): + def test_version(self): + self.assertEqual(sdk_version, vendored_version, "SDK version should match vendored version") + self.assertEqual(exporter_version, vendored_version, "Exporter version should match vendored version")