diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/environment_variables.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/environment_variables.py new file mode 100644 index 0000000000..02bdfe68af --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/environment_variables.py @@ -0,0 +1,3 @@ +OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = ( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT" +) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py index 098572fb08..6798e372b9 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py +++ b/instrumentation/opentelemetry-instrumentation-botocore/src/opentelemetry/instrumentation/botocore/extensions/bedrock.py @@ -18,14 +18,9 @@ from __future__ import annotations -import io -import json import logging -import math from typing import Any -from botocore.response import StreamingBody - from opentelemetry.instrumentation.botocore.extensions.types import ( _AttributeMapT, _AwsSdkExtension, @@ -34,6 +29,7 @@ GEN_AI_OPERATION_NAME, GEN_AI_REQUEST_MAX_TOKENS, GEN_AI_REQUEST_MODEL, + GEN_AI_REQUEST_STOP_SEQUENCES, GEN_AI_REQUEST_TEMPERATURE, GEN_AI_REQUEST_TOP_P, GEN_AI_RESPONSE_FINISH_REASONS, @@ -58,327 +54,78 @@ class _BedrockRuntimeExtension(_AwsSdkExtension): """ def extract_attributes(self, attributes: _AttributeMapT): - attributes[GEN_AI_SYSTEM] = GenAiSystemValues.AWS_BEDROCK - attributes[GEN_AI_OPERATION_NAME] = GenAiOperationNameValues.CHAT + attributes[GEN_AI_SYSTEM] = GenAiSystemValues.AWS_BEDROCK.value model_id = self._call_context.params.get(_MODEL_ID_KEY) if model_id: attributes[GEN_AI_REQUEST_MODEL] = model_id - # Get the request body if it exists - body = self._call_context.params.get("body") - if body: - try: - request_body = json.loads(body) - - if "amazon.titan" in model_id: - self._extract_titan_attributes( - attributes, request_body - ) - if "amazon.nova" in model_id: - self._extract_nova_attributes(attributes, request_body) - elif "anthropic.claude" in model_id: - self._extract_claude_attributes( - attributes, request_body - ) - elif "meta.llama" in model_id: - self._extract_llama_attributes( - attributes, request_body - ) - elif "cohere.command" in model_id: - self._extract_cohere_attributes( - attributes, request_body - ) - elif "ai21.jamba" in model_id: - self._extract_ai21_attributes(attributes, request_body) - elif "mistral" in model_id: - self._extract_mistral_attributes( - attributes, request_body - ) - - except json.JSONDecodeError: - _logger.debug("Error: Unable to parse the body as JSON") - - def _extract_titan_attributes(self, attributes, request_body): - config = request_body.get("textGenerationConfig", {}) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TEMPERATURE, config.get("temperature") - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, config.get("topP") - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_MAX_TOKENS, config.get("maxTokenCount") - ) - - def _extract_nova_attributes(self, attributes, request_body): - config = request_body.get("inferenceConfig", {}) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TEMPERATURE, config.get("temperature") - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, config.get("top_p") - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_MAX_TOKENS, config.get("max_new_tokens") - ) - - def _extract_claude_attributes(self, attributes, request_body): - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_MAX_TOKENS, - request_body.get("max_tokens"), - ) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_TEMPERATURE, - request_body.get("temperature"), - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, request_body.get("top_p") - ) - - def _extract_cohere_attributes(self, attributes, request_body): - prompt = request_body.get("message") - if prompt: - attributes[GEN_AI_USAGE_INPUT_TOKENS] = math.ceil(len(prompt) / 6) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_MAX_TOKENS, - request_body.get("max_tokens"), - ) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_TEMPERATURE, - request_body.get("temperature"), - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, request_body.get("p") - ) - - def _extract_ai21_attributes(self, attributes, request_body): - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_MAX_TOKENS, - request_body.get("max_tokens"), - ) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_TEMPERATURE, - request_body.get("temperature"), - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, request_body.get("top_p") - ) - - def _extract_llama_attributes(self, attributes, request_body): - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_MAX_TOKENS, - request_body.get("max_gen_len"), - ) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_TEMPERATURE, - request_body.get("temperature"), - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, request_body.get("top_p") - ) + # FIXME: add other model patterns + text_model_patterns = ["amazon.titan-text", "anthropic.claude"] + if any(pattern in model_id for pattern in text_model_patterns): + attributes[GEN_AI_OPERATION_NAME] = ( + GenAiOperationNameValues.CHAT.value + ) - def _extract_mistral_attributes(self, attributes, request_body): - prompt = request_body.get("prompt") - if prompt: - attributes[GEN_AI_USAGE_INPUT_TOKENS] = math.ceil(len(prompt) / 6) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_MAX_TOKENS, - request_body.get("max_tokens"), - ) - self._set_if_not_none( - attributes, - GEN_AI_REQUEST_TEMPERATURE, - request_body.get("temperature"), - ) - self._set_if_not_none( - attributes, GEN_AI_REQUEST_TOP_P, request_body.get("top_p") - ) + if inference_config := self._call_context.params.get( + "inferenceConfig" + ): + self._set_if_not_none( + attributes, + GEN_AI_REQUEST_TEMPERATURE, + inference_config.get("temperature"), + ) + self._set_if_not_none( + attributes, + GEN_AI_REQUEST_TOP_P, + inference_config.get("topP"), + ) + self._set_if_not_none( + attributes, + GEN_AI_REQUEST_MAX_TOKENS, + inference_config.get("maxTokens"), + ) + self._set_if_not_none( + attributes, + GEN_AI_REQUEST_STOP_SEQUENCES, + inference_config.get("stopSequences"), + ) @staticmethod def _set_if_not_none(attributes, key, value): if value is not None: attributes[key] = value - # pylint: disable=too-many-branches + def before_service_call(self, span: Span): + if not span.is_recording(): + return + + operation_name = span.attributes.get(GEN_AI_OPERATION_NAME, "") + request_model = span.attributes.get(GEN_AI_REQUEST_MODEL, "") + # avoid setting to an empty string if are not available + if operation_name and request_model: + span.update_name(f"{operation_name} {request_model}") + def on_success(self, span: Span, result: dict[str, Any]): model_id = self._call_context.params.get(_MODEL_ID_KEY) if not model_id: return - if "body" in result and isinstance(result["body"], StreamingBody): - original_body = None - try: - original_body = result["body"] - body_content = original_body.read() - - # Use one stream for telemetry - stream = io.BytesIO(body_content) - telemetry_content = stream.read() - response_body = json.loads(telemetry_content.decode("utf-8")) - if "amazon.titan" in model_id: - self._handle_amazon_titan_response(span, response_body) - if "amazon.nova" in model_id: - self._handle_amazon_nova_response(span, response_body) - elif "anthropic.claude" in model_id: - self._handle_anthropic_claude_response(span, response_body) - elif "meta.llama" in model_id: - self._handle_meta_llama_response(span, response_body) - elif "cohere.command" in model_id: - self._handle_cohere_command_response(span, response_body) - elif "ai21.jamba" in model_id: - self._handle_ai21_jamba_response(span, response_body) - elif "mistral" in model_id: - self._handle_mistral_mistral_response(span, response_body) - # Replenish stream for downstream application use - new_stream = io.BytesIO(body_content) - result["body"] = StreamingBody(new_stream, len(body_content)) - - except json.JSONDecodeError: - _logger.debug( - "Error: Unable to parse the response body as JSON" - ) - except Exception as e: # pylint: disable=broad-exception-caught, invalid-name - _logger.debug("Error processing response: %s", e) - finally: - if original_body is not None: - original_body.close() - - # pylint: disable=no-self-use - def _handle_amazon_titan_response( - self, span: Span, response_body: dict[str, Any] - ): - if "inputTextTokenCount" in response_body: - span.set_attribute( - GEN_AI_USAGE_INPUT_TOKENS, response_body["inputTextTokenCount"] - ) - if "results" in response_body and response_body["results"]: - result = response_body["results"][0] - if "tokenCount" in result: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, result["tokenCount"] - ) - if "completionReason" in result: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, - [result["completionReason"]], - ) - - # pylint: disable=no-self-use - def _handle_amazon_nova_response( - self, span: Span, response_body: dict[str, Any] - ): - if "usage" in response_body: - usage = response_body["usage"] - if "inputTokens" in usage: - span.set_attribute( - GEN_AI_USAGE_INPUT_TOKENS, usage["inputTokens"] - ) - if "outputTokens" in usage: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, usage["outputTokens"] - ) - if "stopReason" in response_body: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, [response_body["stopReason"]] - ) - - # pylint: disable=no-self-use - def _handle_anthropic_claude_response( - self, span: Span, response_body: dict[str, Any] - ): - if "usage" in response_body: - usage = response_body["usage"] - if "input_tokens" in usage: + if usage := result.get("usage"): + if input_tokens := usage.get("inputTokens"): span.set_attribute( - GEN_AI_USAGE_INPUT_TOKENS, usage["input_tokens"] + GEN_AI_USAGE_INPUT_TOKENS, + input_tokens, ) - if "output_tokens" in usage: + if output_tokens := usage.get("outputTokens"): span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, usage["output_tokens"] + GEN_AI_USAGE_OUTPUT_TOKENS, + output_tokens, ) - if "stop_reason" in response_body: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, [response_body["stop_reason"]] - ) - # pylint: disable=no-self-use - def _handle_cohere_command_response( - self, span: Span, response_body: dict[str, Any] - ): - # Output tokens: Approximate from the response text - if "text" in response_body: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, - math.ceil(len(response_body["text"]) / 6), - ) - if "finish_reason" in response_body: + if stop_reason := result.get("stopReason"): span.set_attribute( GEN_AI_RESPONSE_FINISH_REASONS, - [response_body["finish_reason"]], - ) - - # pylint: disable=no-self-use - def _handle_ai21_jamba_response( - self, span: Span, response_body: dict[str, Any] - ): - if "usage" in response_body: - usage = response_body["usage"] - if "prompt_tokens" in usage: - span.set_attribute( - GEN_AI_USAGE_INPUT_TOKENS, usage["prompt_tokens"] - ) - if "completion_tokens" in usage: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, usage["completion_tokens"] - ) - if "choices" in response_body: - choices = response_body["choices"][0] - if "finish_reason" in choices: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, [choices["finish_reason"]] - ) - - # pylint: disable=no-self-use - def _handle_meta_llama_response( - self, span: Span, response_body: dict[str, Any] - ): - if "prompt_token_count" in response_body: - span.set_attribute( - GEN_AI_USAGE_INPUT_TOKENS, response_body["prompt_token_count"] - ) - if "generation_token_count" in response_body: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, - response_body["generation_token_count"], - ) - if "stop_reason" in response_body: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, [response_body["stop_reason"]] - ) - - # pylint: disable=no-self-use - def _handle_mistral_mistral_response( - self, span: Span, response_body: dict[str, Any] - ): - if "outputs" in response_body: - outputs = response_body["outputs"][0] - if "text" in outputs: - span.set_attribute( - GEN_AI_USAGE_OUTPUT_TOKENS, - math.ceil(len(outputs["text"]) / 6), - ) - if "stop_reason" in outputs: - span.set_attribute( - GEN_AI_RESPONSE_FINISH_REASONS, [outputs["stop_reason"]] + [stop_reason], ) diff --git a/instrumentation/opentelemetry-instrumentation-botocore/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-0.txt similarity index 97% rename from instrumentation/opentelemetry-instrumentation-botocore/test-requirements.txt rename to instrumentation/opentelemetry-instrumentation-botocore/test-requirements-0.txt index aa5f89859f..ee28a1f2ba 100644 --- a/instrumentation/opentelemetry-instrumentation-botocore/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-0.txt @@ -19,6 +19,7 @@ pluggy==1.5.0 py-cpuinfo==9.0.0 pycparser==2.21 pytest==7.4.4 +pytest-vcr==1.0.2 python-dateutil==2.8.2 pytz==2024.1 PyYAML==6.0.1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-1.txt b/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-1.txt new file mode 100644 index 0000000000..c4695ff27c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-1.txt @@ -0,0 +1,39 @@ +asgiref==3.8.1 +aws-xray-sdk==2.12.1 +boto3==1.35.56 +botocore==1.35.56 +certifi==2024.7.4 +cffi==1.17.0 +charset-normalizer==3.3.2 +cryptography==43.0.1 +Deprecated==1.2.14 +docker==7.0.0 +idna==3.7 +iniconfig==2.0.0 +Jinja2==3.1.4 +jmespath==1.0.1 +MarkupSafe==2.1.5 +moto==5.0.9 +packaging==24.0 +pluggy==1.5.0 +py-cpuinfo==9.0.0 +pycparser==2.21 +pytest==7.4.4 +pytest-vcr==1.0.2 +python-dateutil==2.8.2 +pytz==2024.1 +PyYAML==6.0.1 +requests==2.32.3 +responses==0.25.0 +s3transfer==0.10.0 +six==1.16.0 +tomli==2.0.1 +typing_extensions==4.12.2 +urllib3==1.26.19 +Werkzeug==3.0.6 +wrapt==1.16.0 +xmltodict==0.13.0 +zipp==3.19.2 +-e opentelemetry-instrumentation +-e propagator/opentelemetry-propagator-aws-xray +-e instrumentation/opentelemetry-instrumentation-botocore diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[amazon.titan].yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[amazon.titan].yaml new file mode 100644 index 0000000000..38b1357819 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[amazon.titan].yaml @@ -0,0 +1,93 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test" + } + ] + } + ], + "inferenceConfig": { + "maxTokens": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + } + } + headers: + Content-Length: + - '170' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS4zNS41NiBtZC9Cb3RvY29yZSMxLjM1LjU2IHVhLzIuMCBvcy9saW51eCM2LjEuMC0x + MDM0LW9lbSBtZC9hcmNoI3g4Nl82NCBsYW5nL3B5dGhvbiMzLjEwLjEyIG1kL3B5aW1wbCNDUHl0 + aG9uIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3RvY29yZS8xLjM1LjU2 + X-Amz-Date: + - !!binary | + MjAyNDEyMzFUMTAwMjI1Wg== + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - !!binary | + Um9vdD0xLTg5Y2FhMmUwLTUzY2NiZTE0MzQ0MGE4MWRjMTgyYzUwNjtQYXJlbnQ9ODdiZTZmMGI4 + YjlhNzQxZDtTYW1wbGVkPTE= + amz-sdk-invocation-id: + - !!binary | + YjE5MzViZmEtOGI3Ni00ODMwLTkzNDktMjZhODJjNTQ3Mjg0 + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.eu-central-1.amazonaws.com/model/amazon.titan-text-lite-v1/converse + response: + body: + string: |- + { + "metrics": { + "latencyMs": 527 + }, + "output": { + "message": { + "content": [ + { + "text": "" + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 8, + "outputTokens": 5, + "totalTokens": 13 + } + } + headers: + Connection: + - keep-alive + Content-Length: + - '179' + Content-Type: + - application/json + Date: + - Tue, 31 Dec 2024 10:02:26 GMT + Set-Cookie: test_set_cookie + x-amzn-RequestId: + - 8e183234-2be0-4d6f-8a0e-ff065d47ea60 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[anthropic.claude].yaml b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[anthropic.claude].yaml new file mode 100644 index 0000000000..756c09453c --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/cassettes/test_converse_with_content[anthropic.claude].yaml @@ -0,0 +1,93 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": [ + { + "text": "Say this is a test" + } + ] + } + ], + "inferenceConfig": { + "maxTokens": 10, + "temperature": 0.8, + "topP": 1, + "stopSequences": [ + "|" + ] + } + } + headers: + Content-Length: + - '170' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS4zNS41NiBtZC9Cb3RvY29yZSMxLjM1LjU2IHVhLzIuMCBvcy9saW51eCM2LjEuMC0x + MDM0LW9lbSBtZC9hcmNoI3g4Nl82NCBsYW5nL3B5dGhvbiMzLjEwLjEyIG1kL3B5aW1wbCNDUHl0 + aG9uIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3RvY29yZS8xLjM1LjU2 + X-Amz-Date: + - !!binary | + MjAyNDEyMzFUMTAwMjI2Wg== + X-Amz-Security-Token: + - test_aws_security_token + X-Amzn-Trace-Id: + - !!binary | + Um9vdD0xLTc5MDNlZTI5LWFjNTViNDljOWVkZmExNDhhNjVjMDgxNjtQYXJlbnQ9OGZjODcxYmIw + NjI1ZTEwNDtTYW1wbGVkPTE= + amz-sdk-invocation-id: + - !!binary | + OTU5ZDg1MWItNzg3Zi00NjI3LTk0MGQtNzk2MjJmYjE0ZjQ4 + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + authorization: + - Bearer test_aws_authorization + method: POST + uri: https://bedrock-runtime.eu-central-1.amazonaws.com/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse + response: + body: + string: |- + { + "metrics": { + "latencyMs": 890 + }, + "output": { + "message": { + "content": [ + { + "text": "This is a test." + } + ], + "role": "assistant" + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 12, + "outputTokens": 8, + "totalTokens": 20 + } + } + headers: + Connection: + - keep-alive + Content-Length: + - '195' + Content-Type: + - application/json + Date: + - Tue, 31 Dec 2024 10:02:27 GMT + Set-Cookie: test_set_cookie + x-amzn-RequestId: + - 3bc52e82-cf3a-42a6-a5d6-e4a4fe3dfbbf + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-botocore/tests/conftest.py new file mode 100644 index 0000000000..271c540da7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/conftest.py @@ -0,0 +1,190 @@ +"""Unit tests configuration module.""" + +import json +import os + +import boto3 +import pytest +import yaml + +from opentelemetry.instrumentation.botocore import BotocoreInstrumentor +from opentelemetry.instrumentation.botocore.environment_variables import ( + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, +) +from opentelemetry.sdk._events import EventLoggerProvider +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogExporter, + SimpleLogRecordProcessor, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +@pytest.fixture(scope="function", name="span_exporter") +def fixture_span_exporter(): + exporter = InMemorySpanExporter() + yield exporter + + +@pytest.fixture(scope="function", name="log_exporter") +def fixture_log_exporter(): + exporter = InMemoryLogExporter() + yield exporter + + +@pytest.fixture(scope="function", name="tracer_provider") +def fixture_tracer_provider(span_exporter): + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + return provider + + +@pytest.fixture(scope="function", name="event_logger_provider") +def fixture_event_logger_provider(log_exporter): + provider = LoggerProvider() + provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) + event_logger_provider = EventLoggerProvider(provider) + + return event_logger_provider + + +@pytest.fixture +def bedrock_runtime_client(): + return boto3.client("bedrock-runtime") + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("AWS_ACCESS_KEY_ID"): + os.environ["AWS_ACCESS_KEY_ID"] = "test_aws_access_key_id" + if not os.getenv("AWS_SECRET_ACCESS_KEY"): + os.environ["AWS_SECRET_ACCESS_KEY"] = "test_aws_secret_key" + if not os.getenv("AWS_SESSION_TOKEN"): + os.environ["AWS_SESSION_TOKEN"] = "test_aws_session_token" + if not os.getenv("AWS_DEFAULT_REGION"): + os.environ["AWS_DEFAULT_REGION"] = "eu-central-1" + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("cookie", "test_cookie"), + ("authorization", "Bearer test_aws_authorization"), + ("X-Amz-Security-Token", "test_aws_security_token"), + ], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } + + +@pytest.fixture(scope="function") +def instrument_no_content(tracer_provider, event_logger_provider): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "False"} + ) + + instrumentor = BotocoreInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +@pytest.fixture(scope="function") +def instrument_with_content(tracer_provider, event_logger_provider): + os.environ.update( + {OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT: "True"} + ) + instrumentor = BotocoreInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + event_logger_provider=event_logger_provider, + ) + + yield instrumentor + os.environ.pop(OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, None) + instrumentor.uninstrument() + + +class LiteralBlockScalar(str): + """Formats the string as a literal block scalar, preserving whitespace and + without interpreting escape characters""" + + +def literal_block_scalar_presenter(dumper, data): + """Represents a scalar string as a literal block, via '|' syntax""" + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + + +yaml.add_representer(LiteralBlockScalar, literal_block_scalar_presenter) + + +def process_string_value(string_value): + """Pretty-prints JSON or returns long strings as a LiteralBlockScalar""" + try: + json_data = json.loads(string_value) + return LiteralBlockScalar(json.dumps(json_data, indent=2)) + except (ValueError, TypeError): + if len(string_value) > 80: + return LiteralBlockScalar(string_value) + return string_value + + +def convert_body_to_literal(data): + """Searches the data for body strings, attempting to pretty-print JSON""" + if isinstance(data, dict): + for key, value in data.items(): + # Handle response body case (e.g., response.body.string) + if key == "body" and isinstance(value, dict) and "string" in value: + value["string"] = process_string_value(value["string"]) + + # Handle request body case (e.g., request.body) + elif key == "body" and isinstance(value, str): + data[key] = process_string_value(value) + + else: + convert_body_to_literal(value) + + elif isinstance(data, list): + for idx, choice in enumerate(data): + data[idx] = convert_body_to_literal(choice) + + return data + + +class PrettyPrintJSONBody: + """This makes request and response body recordings more readable.""" + + @staticmethod + def serialize(cassette_dict): + cassette_dict = convert_body_to_literal(cassette_dict) + return yaml.dump( + cassette_dict, default_flow_style=False, allow_unicode=True + ) + + @staticmethod + def deserialize(cassette_string): + return yaml.load(cassette_string, Loader=yaml.Loader) + + +@pytest.fixture(scope="module", autouse=True) +def fixture_vcr(vcr): + vcr.register_serializer("yaml", PrettyPrintJSONBody) + return vcr + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response diff --git a/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py new file mode 100644 index 0000000000..3de3223af6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-botocore/tests/test_botocore_bedrock.py @@ -0,0 +1,155 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +import boto3 +import pytest + +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + +BOTO3_VERSION = tuple(int(x) for x in boto3.__version__.split(".")) + + +@pytest.mark.skipif( + BOTO3_VERSION < (1, 35, 56), reason="Converse API not available" +) +@pytest.mark.vcr() +@pytest.mark.parametrize("llm_model", ["amazon.titan", "anthropic.claude"]) +def test_converse_with_content( + llm_model, + span_exporter, + log_exporter, + bedrock_runtime_client, + instrument_with_content, +): + llm_model_id = { + "amazon.titan": "amazon.titan-text-lite-v1", + "anthropic.claude": "anthropic.claude-3-haiku-20240307-v1:0", + } + messages = [{"role": "user", "content": [{"text": "Say this is a test"}]}] + + llm_model_value = llm_model_id[llm_model] + max_tokens, temperature, top_p, stop_sequences = 10, 0.8, 1, ["|"] + response = bedrock_runtime_client.converse( + messages=messages, + modelId=llm_model_value, + inferenceConfig={ + "maxTokens": max_tokens, + "temperature": temperature, + "topP": top_p, + "stopSequences": stop_sequences, + }, + ) + + (span,) = span_exporter.get_finished_spans() + assert_completion_attributes( + span, + llm_model_value, + response, + "chat", + top_p, + temperature, + max_tokens, + stop_sequences, + ) + + logs = log_exporter.get_finished_logs() + assert len(logs) == 0 + + +def assert_completion_attributes( + span: ReadableSpan, + request_model: str, + response: dict[str, Any], + operation_name: str = "chat", + request_top_p: int | None = None, + request_temperature: int | None = None, + request_max_tokens: int | None = None, + request_stop_sequences: list[str] | None = None, +): + return assert_all_attributes( + span, + request_model, + response["usage"]["inputTokens"], + response["usage"]["outputTokens"], + (response["stopReason"],), + operation_name, + request_top_p, + request_temperature, + request_max_tokens, + tuple(request_stop_sequences), + ) + + +def assert_equal_or_not_present(value, attribute_name, span): + if value: + assert value == span.attributes[attribute_name] + else: + assert attribute_name not in span.attributes + + +def assert_all_attributes( + span: ReadableSpan, + request_model: str, + input_tokens: int | None = None, + output_tokens: int | None = None, + finish_reason: tuple[str] | None = None, + operation_name: str = "chat", + request_top_p: int | None = None, + request_temperature: int | None = None, + request_max_tokens: int | None = None, + request_stop_sequences: tuple[str] | None = None, +): + assert span.name == f"{operation_name} {request_model}" + assert ( + operation_name + == span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + ) + assert ( + GenAIAttributes.GenAiSystemValues.AWS_BEDROCK.value + == span.attributes[GenAIAttributes.GEN_AI_SYSTEM] + ) + assert ( + request_model == span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + ) + + assert_equal_or_not_present( + input_tokens, GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, span + ) + assert_equal_or_not_present( + output_tokens, GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, span + ) + assert_equal_or_not_present( + finish_reason, GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, span + ) + assert_equal_or_not_present( + request_top_p, GenAIAttributes.GEN_AI_REQUEST_TOP_P, span + ) + assert_equal_or_not_present( + request_temperature, GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE, span + ) + assert_equal_or_not_present( + request_max_tokens, GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS, span + ) + assert_equal_or_not_present( + request_stop_sequences, + GenAIAttributes.GEN_AI_REQUEST_STOP_SEQUENCES, + span, + ) diff --git a/tox.ini b/tox.ini index 6c6abe3d42..7fab44d4fe 100644 --- a/tox.ini +++ b/tox.ini @@ -64,7 +64,7 @@ envlist = lint-instrumentation-aws-lambda ; opentelemetry-instrumentation-botocore - py3{8,9,10,11,12}-test-instrumentation-botocore + py3{8,9,10,11,12}-test-instrumentation-botocore-{0,1} ; FIXME: see https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1736 ; pypy3-test-instrumentation-botocore lint-instrumentation-botocore @@ -407,6 +407,11 @@ test_deps = opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils +pass_env = + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN + AWS_DEFAULT_REGION deps = lint: -r dev-requirements.txt @@ -511,7 +516,9 @@ deps = lint-instrumentation-urllib3: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-urllib3/test-requirements-1.txt botocore: {[testenv]test_deps} - botocore: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore/test-requirements.txt + botocore-0: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-0.txt + botocore-1: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-1.txt + lint-instrumentation-botocore: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-botocore/test-requirements-1.txt cassandra: {[testenv]test_deps} cassandra: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-cassandra/test-requirements.txt