From 2bdb403040a103b651515b1d59bbd6d8a8dd2a4a Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Mon, 2 Dec 2024 23:44:51 +0100 Subject: [PATCH 01/13] feat: add warnings and errors to output impacts object --- ecologits/impacts/modeling.py | 35 ++++++++++++++++++----- ecologits/model_repository.py | 47 +++++++++++++++++++++---------- ecologits/tracers/utils.py | 13 +++++++-- ecologits/warnings_and_errors.py | 39 +++++++++++++++++++++++++ tests/test_warnings_and_errors.py | 47 +++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 25 deletions(-) create mode 100644 ecologits/warnings_and_errors.py create mode 100644 tests/test_warnings_and_errors.py diff --git a/ecologits/impacts/modeling.py b/ecologits/impacts/modeling.py index 68c73dc5..d6f17d09 100644 --- a/ecologits/impacts/modeling.py +++ b/ecologits/impacts/modeling.py @@ -1,9 +1,10 @@ from functools import total_ordering -from typing import TypeVar +from typing import Optional, TypeVar from pydantic import BaseModel from ecologits.exceptions import ModelingError +from ecologits.warnings_and_errors import BaseError, BaseWarning from ecologits.utils.range_value import ValueOrRange Impact = TypeVar("Impact", bound="BaseImpact") @@ -199,9 +200,29 @@ class Impacts(BaseModel): usage: Impacts for the usage phase embodied: Impacts for the embodied phase """ - energy: Energy - gwp: GWP - adpe: ADPe - pe: PE - usage: Usage - embodied: Embodied + energy: Optional[Energy] = None + gwp: Optional[GWP] = None + adpe: Optional[ADPe] = None + pe: Optional[PE] = None + usage: Optional[Usage] = None + embodied: Optional[Embodied] = None + warnings: list[BaseWarning] = [] + errors: list[BaseError] = [] + + @property + def has_warnings(self) -> bool: + return len(self.warnings) > 0 + + @property + def has_errors(self) -> bool: + return len(self.errors) > 0 + + def add_warning(self, warning: BaseWarning) -> None: + if self.warnings is None: + self.warnings = [] + self.warnings.append(warning) + + def add_errors(self, error: BaseError) -> None: + if self.errors is None: + self.errors = [] + self.errors.append(error) diff --git a/ecologits/model_repository.py b/ecologits/model_repository.py index 7cfad301..a6fdd9f3 100644 --- a/ecologits/model_repository.py +++ b/ecologits/model_repository.py @@ -1,10 +1,11 @@ import json import os from enum import Enum -from typing import Optional, Union +from typing import Any, Optional, Union from pydantic import BaseModel +from ecologits.warnings_and_errors import BaseWarning from ecologits.utils.range_value import ValueOrRange @@ -17,12 +18,6 @@ class Providers(Enum): google = "google" -class Warnings(Enum): - model_architecture_not_released = "model_architecture_not_released" - model_architecture_multimodal = "model_architecture_multimodal" - - - class ArchitectureTypes(Enum): DENSE = "dense" MOE = "moe" @@ -48,13 +43,25 @@ class Model(BaseModel): provider: Providers name: str architecture: Architecture - warnings: Optional[list[Warnings]] = None - sources: Optional[list[str]] = None + warnings: list[BaseWarning] = [] + sources: list[str] = [] + @property + def has_warnings(self) -> bool: + return len(self.warnings) > 0 -class Models(BaseModel): - aliases: Optional[list[Alias]] = None - models: Optional[list[Model]] = None + @classmethod + def from_json(cls, data: dict[str, Any]) -> "Model": + warnings = [] + if data["warnings"] is not None: + warnings = [BaseWarning.from_code(code) for code in data["warnings"]] + return cls( + provider=Providers(data["provider"]), + name=data["name"], + architecture=Architecture.model_validate(data["architecture"]), + warnings=warnings, + sources=data["sources"] or [] + ) class ModelRepository: @@ -92,10 +99,20 @@ def from_json(cls, filepath: Optional[str] = None) -> "ModelRepository": ) with open(filepath) as fd: data = json.load(fd) - mf = Models.model_validate(data) - if mf.models is None: + + alias_list = [] + if "aliases" in data and data["aliases"] is not None: + for alias in data["aliases"]: + alias_list.append(Alias.model_validate(alias)) + + model_list = [] + if "models" in data and data["models"] is not None: + for model in data["models"]: + model_list.append(Model.from_json(model)) + + if len(model_list) == 0: raise ValueError("Cannot initialize on an empty model repository.") - return cls(models=mf.models, aliases=mf.aliases) + return cls(models=model_list, aliases=alias_list) models = ModelRepository.from_json() diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index d6524db4..e9994fa1 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -4,6 +4,7 @@ from ecologits.impacts.llm import compute_llm_impacts from ecologits.impacts.modeling import Impacts from ecologits.log import logger +from ecologits.warnings_and_errors import ModelNotRegisteredError, ZoneDoesNotExistError from ecologits.model_repository import ParametersMoE, models @@ -35,7 +36,7 @@ def llm_impacts( model = models.find_model(provider=provider, model_name=model_name) if model is None: logger.debug(f"Could not find model `{model_name}` for {provider} provider.") - return None + return Impacts(errors=[ModelNotRegisteredError()]) if isinstance(model.architecture.parameters, ParametersMoE): model_total_params = model.architecture.parameters.total @@ -47,11 +48,11 @@ def llm_impacts( electricity_mix = electricity_mixes.find_electricity_mix(zone=electricity_mix_zone) if electricity_mix is None: logger.debug(f"Could not find electricity mix `{electricity_mix_zone}` in the ADEME database") - return None + return Impacts(errors=[ZoneDoesNotExistError()]) if_electricity_mix_adpe=electricity_mix.adpe if_electricity_mix_pe=electricity_mix.pe if_electricity_mix_gwp=electricity_mix.gwp - return compute_llm_impacts( + impacts = compute_llm_impacts( model_active_parameter_count=model_active_params, model_total_parameter_count=model_total_params, output_token_count=output_token_count, @@ -60,3 +61,9 @@ def llm_impacts( if_electricity_mix_pe=if_electricity_mix_pe, if_electricity_mix_gwp=if_electricity_mix_gwp, ) + + if model.has_warnings: + for w in model.warnings: + impacts.add_warning(w) + + return impacts diff --git a/ecologits/warnings_and_errors.py b/ecologits/warnings_and_errors.py new file mode 100644 index 00000000..b3764087 --- /dev/null +++ b/ecologits/warnings_and_errors.py @@ -0,0 +1,39 @@ +from typing import Optional + +from pydantic import BaseModel + + +class BaseWarning(BaseModel): + text: Optional[str] = None + + @classmethod + def from_code(cls, code: str) -> "BaseWarning": + if code not in _warning_codes: + raise ValueError(f"Unknown annotation code: {code}") + return _warning_codes[code]() + + +class BaseError(BaseModel): + text: Optional[str] = None + + +class ModelArchNotReleasedWarning(BaseWarning): + text: str = "The model architecture has not been released, expect lower precision." + + +class ModelArchMultimodalWarning(BaseWarning): + text: str = "The model architecture is multimodal, expect lower precision." + + +class ModelNotRegisteredError(BaseError): + text: str = "The model is not registered in the model repository." + + +class ZoneDoesNotExistError(BaseError): + text: str = "The zone does not exist." + + +_warning_codes: dict[str, type[BaseWarning]] = { + "model_architecture_not_released": ModelArchNotReleasedWarning, + "model_architecture_multimodal": ModelArchMultimodalWarning +} diff --git a/tests/test_warnings_and_errors.py b/tests/test_warnings_and_errors.py new file mode 100644 index 00000000..57942d6f --- /dev/null +++ b/tests/test_warnings_and_errors.py @@ -0,0 +1,47 @@ +import pytest + +from ecologits.tracers.utils import llm_impacts +from ecologits.warnings_and_errors import ( + ModelArchNotReleasedWarning, + ModelArchMultimodalWarning, + ModelNotRegisteredError, + ZoneDoesNotExistError +) + + +def test_warnings(): + impacts = llm_impacts( + provider="openai", + model_name="gpt-4o-mini", + output_token_count=10, + request_latency=10 + ) + assert impacts.energy.value > 0 + assert impacts.has_warnings + assert isinstance(impacts.warnings[0], (ModelArchNotReleasedWarning, ModelArchMultimodalWarning)) + assert isinstance(impacts.warnings[1], (ModelArchNotReleasedWarning, ModelArchMultimodalWarning)) + + +def test_model_error(): + impacts = llm_impacts( + provider="openai", + model_name="unknown-model", + output_token_count=10, + request_latency=10 + ) + assert impacts.energy is None + assert impacts.has_errors + assert isinstance(impacts.errors[0], ModelNotRegisteredError) + + +def test_zone_error(): + impacts = llm_impacts( + provider="openai", + model_name="gpt-4o-mini", + output_token_count=10, + request_latency=10, + electricity_mix_zone="UNKNOWN-ZONE" + ) + assert impacts.energy is None + assert impacts.has_errors + assert isinstance(impacts.errors[0], ZoneDoesNotExistError) From 27d6075ebfcde3126bad7d835489fd18adcbb645 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Mon, 2 Dec 2024 23:49:55 +0100 Subject: [PATCH 02/13] chore: pre-commit --- ecologits/impacts/modeling.py | 2 +- ecologits/model_repository.py | 2 +- ecologits/tracers/utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ecologits/impacts/modeling.py b/ecologits/impacts/modeling.py index d6f17d09..d1e31353 100644 --- a/ecologits/impacts/modeling.py +++ b/ecologits/impacts/modeling.py @@ -4,8 +4,8 @@ from pydantic import BaseModel from ecologits.exceptions import ModelingError -from ecologits.warnings_and_errors import BaseError, BaseWarning from ecologits.utils.range_value import ValueOrRange +from ecologits.warnings_and_errors import BaseError, BaseWarning Impact = TypeVar("Impact", bound="BaseImpact") diff --git a/ecologits/model_repository.py b/ecologits/model_repository.py index a6fdd9f3..bb8f2622 100644 --- a/ecologits/model_repository.py +++ b/ecologits/model_repository.py @@ -5,8 +5,8 @@ from pydantic import BaseModel -from ecologits.warnings_and_errors import BaseWarning from ecologits.utils.range_value import ValueOrRange +from ecologits.warnings_and_errors import BaseWarning class Providers(Enum): diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index e9994fa1..cab7e16f 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -4,8 +4,8 @@ from ecologits.impacts.llm import compute_llm_impacts from ecologits.impacts.modeling import Impacts from ecologits.log import logger -from ecologits.warnings_and_errors import ModelNotRegisteredError, ZoneDoesNotExistError from ecologits.model_repository import ParametersMoE, models +from ecologits.warnings_and_errors import ModelNotRegisteredError, ZoneDoesNotExistError def _avg(value_range: tuple) -> float: From acc727854aa4dd61c809fa98cae8e0f75e33d570 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Tue, 3 Dec 2024 15:55:52 +0100 Subject: [PATCH 03/13] feat: change naming to alerts + add docs url --- docs/tutorial/alerts.md | 15 ++ ecologits/alerts.py | 51 ++++++ ecologits/data/models.json | 148 +++++++++--------- ecologits/impacts/modeling.py | 10 +- ecologits/model_repository.py | 6 +- ecologits/tracers/utils.py | 4 +- ecologits/warnings_and_errors.py | 39 ----- mkdocs.yml | 1 + ..._warnings_and_errors.py => test_alerts.py} | 6 +- 9 files changed, 154 insertions(+), 126 deletions(-) create mode 100644 docs/tutorial/alerts.md create mode 100644 ecologits/alerts.py delete mode 100644 ecologits/warnings_and_errors.py rename tests/{test_warnings_and_errors.py => test_alerts.py} (89%) diff --git a/docs/tutorial/alerts.md b/docs/tutorial/alerts.md new file mode 100644 index 00000000..90c7c2a5 --- /dev/null +++ b/docs/tutorial/alerts.md @@ -0,0 +1,15 @@ +# Warnings and Errors + +## Warnings + +### `model-arch-not-released` + + +### `model-arch-multimodal` + + +## Errors + +### `model-not-registered` + +### `zone-not-registerd` diff --git a/ecologits/alerts.py b/ecologits/alerts.py new file mode 100644 index 00000000..a553c26e --- /dev/null +++ b/ecologits/alerts.py @@ -0,0 +1,51 @@ +from pydantic import BaseModel + +ALERT_DOCS_URL = "https://ecologits/tutorial/alerts/#{code}" + + +class AlertMessage(BaseModel): + code: str + message: str + + def __str__(self) -> str: + return f"{self.message}\n\nFor further information visit {ALERT_DOCS_URL.format(code=self.code)}" + + @classmethod + def from_code(cls, code: str) -> "AlertMessage": + if code in _warning_codes: + return _warning_codes[code]() + elif code in _error_codes: + return _error_codes[code]() + else: + raise ValueError(f"Alert code `{code}` does not exist.") + + +class ModelArchNotReleasedWarning(AlertMessage): + code: str = "model-arch-not-released" + message: str = "The model architecture has not been released, expect lower precision." + + +class ModelArchMultimodalWarning(AlertMessage): + code: str = "model-arch-multimodal" + message: str = "The model architecture is multimodal, expect lower precision." + + +class ModelNotRegisteredError(AlertMessage): + code: str = "model-not-registered" + message: str = "The model is not registered in the model repository." + + +class ZoneNotRegisteredError(AlertMessage): + code: str = "zone-not-registered" + message: str = "The zone is not registered." + + +_warning_codes: dict[str, type[AlertMessage]] = { + "model-arch-not-released": ModelArchNotReleasedWarning, + "model-arch-multimodal": ModelArchMultimodalWarning +} + +_error_codes: dict[str, type[AlertMessage]] = { + "model-not-registered": ModelNotRegisteredError, + "zone-not-registered": ZoneNotRegisteredError +} diff --git a/ecologits/data/models.json b/ecologits/data/models.json index 3f6476e8..3f6fee9b 100644 --- a/ecologits/data/models.json +++ b/ecologits/data/models.json @@ -50,7 +50,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -68,7 +68,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -86,7 +86,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -104,7 +104,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -122,7 +122,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -140,7 +140,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://platform.openai.com/docs/models/gpt-3-5-turbo" @@ -161,8 +161,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -183,8 +183,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -205,8 +205,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -227,8 +227,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -249,8 +249,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -271,8 +271,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -293,8 +293,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4-turbo-and-gpt-4" @@ -315,8 +315,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o" @@ -337,8 +337,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o" @@ -359,8 +359,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o" @@ -381,8 +381,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o" @@ -400,8 +400,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o-mini" @@ -419,8 +419,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://platform.openai.com/docs/models/gpt-4o-mini" @@ -528,7 +528,7 @@ "parameters": 7.3 }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -543,7 +543,7 @@ "parameters": 7.3 }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -558,7 +558,7 @@ "parameters": 7.3 }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -573,7 +573,7 @@ "parameters": 7.3 }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -591,7 +591,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -609,7 +609,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -627,7 +627,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -645,7 +645,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -669,7 +669,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -693,7 +693,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -717,7 +717,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -738,7 +738,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -759,7 +759,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -780,7 +780,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -801,7 +801,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -822,7 +822,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.mistral.ai/getting-started/models/" @@ -840,7 +840,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -858,7 +858,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -876,7 +876,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -897,7 +897,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -918,7 +918,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -939,7 +939,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -960,7 +960,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://docs.anthropic.com/en/docs/about-claude/models" @@ -1090,7 +1090,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1108,7 +1108,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1126,7 +1126,7 @@ } }, "warnings": [ - "model_architecture_not_released" + "model-arch-not-released" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1144,8 +1144,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1163,8 +1163,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1185,8 +1185,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1207,8 +1207,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1226,8 +1226,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -1245,8 +1245,8 @@ } }, "warnings": [ - "model_architecture_not_released", - "model_architecture_multimodal" + "model-arch-not-released", + "model-arch-multimodal" ], "sources": [ "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" @@ -3357,7 +3357,7 @@ "parameters": 4.15 }, "warnings": [ - "model_architecture_multimodal" + "model-arch-multimodal" ], "sources": [ "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct" @@ -3515,7 +3515,7 @@ "parameters": 4.15 }, "warnings": [ - "model_architecture_multimodal" + "model-arch-multimodal" ], "sources": [ "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct-onnx-cpu" @@ -3530,7 +3530,7 @@ "parameters": 4.15 }, "warnings": [ - "model_architecture_multimodal" + "model-arch-multimodal" ], "sources": [ "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct-onnx-cuda" @@ -3545,7 +3545,7 @@ "parameters": 4.15 }, "warnings": [ - "model_architecture_multimodal" + "model-arch-multimodal" ], "sources": [ "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct-onnx-directml" diff --git a/ecologits/impacts/modeling.py b/ecologits/impacts/modeling.py index d1e31353..a29add6b 100644 --- a/ecologits/impacts/modeling.py +++ b/ecologits/impacts/modeling.py @@ -3,9 +3,9 @@ from pydantic import BaseModel +from ecologits.alerts import AlertMessage from ecologits.exceptions import ModelingError from ecologits.utils.range_value import ValueOrRange -from ecologits.warnings_and_errors import BaseError, BaseWarning Impact = TypeVar("Impact", bound="BaseImpact") @@ -206,8 +206,8 @@ class Impacts(BaseModel): pe: Optional[PE] = None usage: Optional[Usage] = None embodied: Optional[Embodied] = None - warnings: list[BaseWarning] = [] - errors: list[BaseError] = [] + warnings: list[AlertMessage] = [] + errors: list[AlertMessage] = [] @property def has_warnings(self) -> bool: @@ -217,12 +217,12 @@ def has_warnings(self) -> bool: def has_errors(self) -> bool: return len(self.errors) > 0 - def add_warning(self, warning: BaseWarning) -> None: + def add_warning(self, warning: AlertMessage) -> None: if self.warnings is None: self.warnings = [] self.warnings.append(warning) - def add_errors(self, error: BaseError) -> None: + def add_errors(self, error: AlertMessage) -> None: if self.errors is None: self.errors = [] self.errors.append(error) diff --git a/ecologits/model_repository.py b/ecologits/model_repository.py index bb8f2622..bc6951d9 100644 --- a/ecologits/model_repository.py +++ b/ecologits/model_repository.py @@ -5,8 +5,8 @@ from pydantic import BaseModel +from ecologits.alerts import AlertMessage from ecologits.utils.range_value import ValueOrRange -from ecologits.warnings_and_errors import BaseWarning class Providers(Enum): @@ -43,7 +43,7 @@ class Model(BaseModel): provider: Providers name: str architecture: Architecture - warnings: list[BaseWarning] = [] + warnings: list[AlertMessage] = [] sources: list[str] = [] @property @@ -54,7 +54,7 @@ def has_warnings(self) -> bool: def from_json(cls, data: dict[str, Any]) -> "Model": warnings = [] if data["warnings"] is not None: - warnings = [BaseWarning.from_code(code) for code in data["warnings"]] + warnings = [AlertMessage.from_code(code) for code in data["warnings"]] return cls( provider=Providers(data["provider"]), name=data["name"], diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index cab7e16f..f9419dbb 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -1,11 +1,11 @@ from typing import Optional +from ecologits.alerts import ModelNotRegisteredError, ZoneNotRegisteredError from ecologits.electricity_mix_repository import electricity_mixes from ecologits.impacts.llm import compute_llm_impacts from ecologits.impacts.modeling import Impacts from ecologits.log import logger from ecologits.model_repository import ParametersMoE, models -from ecologits.warnings_and_errors import ModelNotRegisteredError, ZoneDoesNotExistError def _avg(value_range: tuple) -> float: @@ -48,7 +48,7 @@ def llm_impacts( electricity_mix = electricity_mixes.find_electricity_mix(zone=electricity_mix_zone) if electricity_mix is None: logger.debug(f"Could not find electricity mix `{electricity_mix_zone}` in the ADEME database") - return Impacts(errors=[ZoneDoesNotExistError()]) + return Impacts(errors=[ZoneNotRegisteredError()]) if_electricity_mix_adpe=electricity_mix.adpe if_electricity_mix_pe=electricity_mix.pe if_electricity_mix_gwp=electricity_mix.gwp diff --git a/ecologits/warnings_and_errors.py b/ecologits/warnings_and_errors.py deleted file mode 100644 index b3764087..00000000 --- a/ecologits/warnings_and_errors.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel - - -class BaseWarning(BaseModel): - text: Optional[str] = None - - @classmethod - def from_code(cls, code: str) -> "BaseWarning": - if code not in _warning_codes: - raise ValueError(f"Unknown annotation code: {code}") - return _warning_codes[code]() - - -class BaseError(BaseModel): - text: Optional[str] = None - - -class ModelArchNotReleasedWarning(BaseWarning): - text: str = "The model architecture has not been released, expect lower precision." - - -class ModelArchMultimodalWarning(BaseWarning): - text: str = "The model architecture is multimodal, expect lower precision." - - -class ModelNotRegisteredError(BaseError): - text: str = "The model is not registered in the model repository." - - -class ZoneDoesNotExistError(BaseError): - text: str = "The zone does not exist." - - -_warning_codes: dict[str, type[BaseWarning]] = { - "model_architecture_not_released": ModelArchNotReleasedWarning, - "model_architecture_multimodal": ModelArchMultimodalWarning -} diff --git a/mkdocs.yml b/mkdocs.yml index 8a7c2874..772764d2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - 'Introduction': tutorial/index.md - 'Environmental Impacts': tutorial/impacts.md - 'Supported providers': tutorial/providers.md + - 'Warnings and Errors': tutorial/alerts.md - 'Providers': - 'Anthropic': tutorial/providers/anthropic.md - 'Cohere': tutorial/providers/cohere.md diff --git a/tests/test_warnings_and_errors.py b/tests/test_alerts.py similarity index 89% rename from tests/test_warnings_and_errors.py rename to tests/test_alerts.py index 57942d6f..8219d4d4 100644 --- a/tests/test_warnings_and_errors.py +++ b/tests/test_alerts.py @@ -1,11 +1,11 @@ import pytest from ecologits.tracers.utils import llm_impacts -from ecologits.warnings_and_errors import ( +from ecologits.alerts import ( ModelArchNotReleasedWarning, ModelArchMultimodalWarning, ModelNotRegisteredError, - ZoneDoesNotExistError + ZoneNotRegisteredError ) @@ -44,4 +44,4 @@ def test_zone_error(): ) assert impacts.energy is None assert impacts.has_errors - assert isinstance(impacts.errors[0], ZoneDoesNotExistError) + assert isinstance(impacts.errors[0], ZoneNotRegisteredError) From 062bc2f301cc18f02eb430fbff61aead0e4f8c78 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Thu, 5 Dec 2024 19:59:06 +0100 Subject: [PATCH 04/13] docs: add more documentation of warnings and errors --- docs/tutorial/alerts.md | 40 +++++++++++++++++++++++++++++++++++++++- ecologits/alerts.py | 7 +++++++ mkdocs.yml | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/docs/tutorial/alerts.md b/docs/tutorial/alerts.md index 90c7c2a5..1650fedd 100644 --- a/docs/tutorial/alerts.md +++ b/docs/tutorial/alerts.md @@ -1,15 +1,53 @@ # Warnings and Errors +EcoLogits may encounter situations where the calculation of environmental impacts has **high risk of inaccuracies or uncertainties** (reported as **warnings**), or where the calculation **fails** due to certain reasons like misconfiguration (reported as **errors**). + +Warnings and errors are reported in the [`Impacts`][impacts.modeling.Impacts] pydantic model within the `warnings` and `errors` fields respectively. Each warning or error contains a `code` (all listed below) and a `message` explaining the issue. + +!!! note "Silent reporting of warnings and errors" + + By default, warnings and errors are reported **silently**. This means you won't see any warning logged or exception raised. This approach ensures your program continues to execute and avoids spamming the log output, especially when executing many requests. + +Code example on how to determine **if your request resulted in any warnings or errors** and how to retrieve them. + +```python +from ecologits import EcoLogits + +EcoLogits.init() + +response = ... # Request code goes here + +if response.impacts.has_warnings: + for w in response.impacts.warnings: + print(w) + +if response.impacts.has_errors: + for e in response.impacts.errors: + print(e) +``` + + ## Warnings +List of all the warnings that EcoLogits can report. + ### `model-arch-not-released` +This warning is reported when the model architecture is not disclosed by the provider. Thus, the estimation of environmental impacts is based on a assumption of the model architecture (e.g. dense or mixture of experts, number of parameters). ### `model-arch-multimodal` +This warning is reported when the model is multimodal. EcoLogits uses energy benchmarking data from open source LLMs that can only generate text. Models that can generate (or use as input) data from other modalities such as image, audio or video are currently not fully supported. + ## Errors +List of all the errors that EcoLogits can report. + ### `model-not-registered` -### `zone-not-registerd` +This error is reported when the selected model is not registered. This can happen when the model has been released recently or if you are using a custom model (such as fine-tuned models). In the first case you can try updating EcoLogits to the latest version, if the error persists, you can [open up an issue :octicons-link-external-16:](https://github.com/genai-impact/ecologits/issues/new?assignees=&labels=bug&projects=&template=bug_report.yml). + +### `zone-not-registered` + +This error is reported when the selected geographical zone is not registered. This can happen if the configured zone does not exist or if the custom zone is not properly registered. diff --git a/ecologits/alerts.py b/ecologits/alerts.py index a553c26e..39f18c5c 100644 --- a/ecologits/alerts.py +++ b/ecologits/alerts.py @@ -4,6 +4,13 @@ class AlertMessage(BaseModel): + """ + Base alert message used for warnings or errors. + + Attributes: + code: Alert code. + message: Message explaining the alert. + """ code: str message: str diff --git a/mkdocs.yml b/mkdocs.yml index 772764d2..84dd89a7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,8 +13,8 @@ nav: - 'Tutorial': - 'Introduction': tutorial/index.md - 'Environmental Impacts': tutorial/impacts.md - - 'Supported providers': tutorial/providers.md - 'Warnings and Errors': tutorial/alerts.md + - 'Supported providers': tutorial/providers.md - 'Providers': - 'Anthropic': tutorial/providers/anthropic.md - 'Cohere': tutorial/providers/cohere.md From 5cc3607cd0b1f4a6a8b343fc1f7d2f600407dae6 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Sat, 7 Dec 2024 15:29:33 +0100 Subject: [PATCH 05/13] refactor: change alerts to status messages and split impacts and output --- .../{alerts.md => warnings_and_errors.md} | 2 +- ecologits/alerts.py | 58 --------------- ecologits/impacts/modeling.py | 35 ++------- ecologits/model_repository.py | 6 +- ecologits/status_messages.py | 73 +++++++++++++++++++ ecologits/tracers/anthropic_tracer.py | 9 +-- ecologits/tracers/cohere_tracer.py | 7 +- ecologits/tracers/huggingface_tracer.py | 7 +- ecologits/tracers/litellm_tracer.py | 7 +- ecologits/tracers/mistralai_tracer_v0.py | 7 +- ecologits/tracers/mistralai_tracer_v1.py | 7 +- ecologits/tracers/openai_tracer.py | 7 +- ecologits/tracers/utils.py | 55 ++++++++++++-- mkdocs.yml | 2 +- ...test_alerts.py => test_status_messages.py} | 4 +- 15 files changed, 156 insertions(+), 130 deletions(-) rename docs/tutorial/{alerts.md => warnings_and_errors.md} (90%) delete mode 100644 ecologits/alerts.py create mode 100644 ecologits/status_messages.py rename tests/{test_alerts.py => test_status_messages.py} (96%) diff --git a/docs/tutorial/alerts.md b/docs/tutorial/warnings_and_errors.md similarity index 90% rename from docs/tutorial/alerts.md rename to docs/tutorial/warnings_and_errors.md index 1650fedd..14b861ff 100644 --- a/docs/tutorial/alerts.md +++ b/docs/tutorial/warnings_and_errors.md @@ -2,7 +2,7 @@ EcoLogits may encounter situations where the calculation of environmental impacts has **high risk of inaccuracies or uncertainties** (reported as **warnings**), or where the calculation **fails** due to certain reasons like misconfiguration (reported as **errors**). -Warnings and errors are reported in the [`Impacts`][impacts.modeling.Impacts] pydantic model within the `warnings` and `errors` fields respectively. Each warning or error contains a `code` (all listed below) and a `message` explaining the issue. +Warnings and errors are reported in the [`ImpactsOutput`][tracers.utils.ImpactsOutput] pydantic model within the `warnings` and `errors` fields respectively. Each warning or error contains a `code` (all listed below) and a `message` explaining the issue. !!! note "Silent reporting of warnings and errors" diff --git a/ecologits/alerts.py b/ecologits/alerts.py deleted file mode 100644 index 39f18c5c..00000000 --- a/ecologits/alerts.py +++ /dev/null @@ -1,58 +0,0 @@ -from pydantic import BaseModel - -ALERT_DOCS_URL = "https://ecologits/tutorial/alerts/#{code}" - - -class AlertMessage(BaseModel): - """ - Base alert message used for warnings or errors. - - Attributes: - code: Alert code. - message: Message explaining the alert. - """ - code: str - message: str - - def __str__(self) -> str: - return f"{self.message}\n\nFor further information visit {ALERT_DOCS_URL.format(code=self.code)}" - - @classmethod - def from_code(cls, code: str) -> "AlertMessage": - if code in _warning_codes: - return _warning_codes[code]() - elif code in _error_codes: - return _error_codes[code]() - else: - raise ValueError(f"Alert code `{code}` does not exist.") - - -class ModelArchNotReleasedWarning(AlertMessage): - code: str = "model-arch-not-released" - message: str = "The model architecture has not been released, expect lower precision." - - -class ModelArchMultimodalWarning(AlertMessage): - code: str = "model-arch-multimodal" - message: str = "The model architecture is multimodal, expect lower precision." - - -class ModelNotRegisteredError(AlertMessage): - code: str = "model-not-registered" - message: str = "The model is not registered in the model repository." - - -class ZoneNotRegisteredError(AlertMessage): - code: str = "zone-not-registered" - message: str = "The zone is not registered." - - -_warning_codes: dict[str, type[AlertMessage]] = { - "model-arch-not-released": ModelArchNotReleasedWarning, - "model-arch-multimodal": ModelArchMultimodalWarning -} - -_error_codes: dict[str, type[AlertMessage]] = { - "model-not-registered": ModelNotRegisteredError, - "zone-not-registered": ZoneNotRegisteredError -} diff --git a/ecologits/impacts/modeling.py b/ecologits/impacts/modeling.py index a29add6b..68c73dc5 100644 --- a/ecologits/impacts/modeling.py +++ b/ecologits/impacts/modeling.py @@ -1,9 +1,8 @@ from functools import total_ordering -from typing import Optional, TypeVar +from typing import TypeVar from pydantic import BaseModel -from ecologits.alerts import AlertMessage from ecologits.exceptions import ModelingError from ecologits.utils.range_value import ValueOrRange @@ -200,29 +199,9 @@ class Impacts(BaseModel): usage: Impacts for the usage phase embodied: Impacts for the embodied phase """ - energy: Optional[Energy] = None - gwp: Optional[GWP] = None - adpe: Optional[ADPe] = None - pe: Optional[PE] = None - usage: Optional[Usage] = None - embodied: Optional[Embodied] = None - warnings: list[AlertMessage] = [] - errors: list[AlertMessage] = [] - - @property - def has_warnings(self) -> bool: - return len(self.warnings) > 0 - - @property - def has_errors(self) -> bool: - return len(self.errors) > 0 - - def add_warning(self, warning: AlertMessage) -> None: - if self.warnings is None: - self.warnings = [] - self.warnings.append(warning) - - def add_errors(self, error: AlertMessage) -> None: - if self.errors is None: - self.errors = [] - self.errors.append(error) + energy: Energy + gwp: GWP + adpe: ADPe + pe: PE + usage: Usage + embodied: Embodied diff --git a/ecologits/model_repository.py b/ecologits/model_repository.py index bc6951d9..afe6a740 100644 --- a/ecologits/model_repository.py +++ b/ecologits/model_repository.py @@ -5,7 +5,7 @@ from pydantic import BaseModel -from ecologits.alerts import AlertMessage +from ecologits.status_messages import WarningMessage from ecologits.utils.range_value import ValueOrRange @@ -43,7 +43,7 @@ class Model(BaseModel): provider: Providers name: str architecture: Architecture - warnings: list[AlertMessage] = [] + warnings: list[WarningMessage] = [] sources: list[str] = [] @property @@ -54,7 +54,7 @@ def has_warnings(self) -> bool: def from_json(cls, data: dict[str, Any]) -> "Model": warnings = [] if data["warnings"] is not None: - warnings = [AlertMessage.from_code(code) for code in data["warnings"]] + warnings = [WarningMessage.from_code(code) for code in data["warnings"]] return cls( provider=Providers(data["provider"]), name=data["name"], diff --git a/ecologits/status_messages.py b/ecologits/status_messages.py new file mode 100644 index 00000000..1e90152c --- /dev/null +++ b/ecologits/status_messages.py @@ -0,0 +1,73 @@ +from pydantic import BaseModel + +STATUS_DOCS_URL = "https://ecologits/tutorial/warnings_and_errors/#{code}" + + +class _StatusMessage(BaseModel): + """ + Base status message used for warnings or errors. + + Attributes: + code: Status code. + message: Message explaining the issue. + """ + code: str + message: str + + def __str__(self) -> str: + return f"{self.message}\n\nFor further information visit {STATUS_DOCS_URL.format(code=self.code)}" + + @classmethod + def from_code(cls, code: str) -> type["_StatusMessage"]: + raise NotImplementedError("Should be called from WarningMessage or ErrorMessage.") + + +class WarningMessage(_StatusMessage): + + @classmethod + def from_code(cls, code: str) -> "WarningMessage": + if code in _warning_codes: + return _warning_codes[code]() + else: + raise ValueError(f"Warning code `{code}` does not exist.") + + +class ErrorMessage(_StatusMessage): + + @classmethod + def from_code(cls, code: str) -> "ErrorMessage": + if code in _error_codes: + return _error_codes[code]() + else: + raise ValueError(f"Error code `{code}` does not exist.") + + +class ModelArchNotReleasedWarning(WarningMessage): + code: str = "model-arch-not-released" + message: str = "The model architecture has not been released, expect lower precision." + + +class ModelArchMultimodalWarning(WarningMessage): + code: str = "model-arch-multimodal" + message: str = "The model architecture is multimodal, expect lower precision." + + +class ModelNotRegisteredError(ErrorMessage): + code: str = "model-not-registered" + message: str = "The model is not registered in the model repository." + + +class ZoneNotRegisteredError(ErrorMessage): + code: str = "zone-not-registered" + message: str = "The zone is not registered." + + +_warning_codes: dict[str, type[WarningMessage]] = { + "model-arch-not-released": ModelArchNotReleasedWarning, + "model-arch-multimodal": ModelArchMultimodalWarning +} + +_error_codes: dict[str, type[ErrorMessage]] = { + "model-not-registered": ModelNotRegisteredError, + "zone-not-registered": ZoneNotRegisteredError +} diff --git a/ecologits/tracers/anthropic_tracer.py b/ecologits/tracers/anthropic_tracer.py index 3c018c7f..6372147f 100644 --- a/ecologits/tracers/anthropic_tracer.py +++ b/ecologits/tracers/anthropic_tracer.py @@ -13,8 +13,7 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "anthropic" @@ -23,11 +22,11 @@ class Message(_Message): - impacts: Impacts + impacts: ImpactsOutput class MessageStream(_MessageStream): - impacts: Optional[Impacts] = None + impacts: Optional[ImpactsOutput] = None @override def __stream_text__(self) -> Iterator[str]: # type: ignore[misc] @@ -62,7 +61,7 @@ def __init__(self, parent) -> None: # noqa: ANN001 class AsyncMessageStream(_AsyncMessageStream): - impacts: Optional[Impacts] = None + impacts: Optional[ImpactsOutput] = None @override async def __stream_text__(self) -> AsyncIterator[str]: # type: ignore[misc] diff --git a/ecologits/tracers/cohere_tracer.py b/ecologits/tracers/cohere_tracer.py index d395dd22..a5ba4af7 100644 --- a/ecologits/tracers/cohere_tracer.py +++ b/ecologits/tracers/cohere_tracer.py @@ -9,21 +9,20 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "cohere" class NonStreamedChatResponse(_NonStreamedChatResponse): - impacts: Optional[Impacts] = None + impacts: Optional[ImpactsOutput] = None class Config: arbitrary_types_allowed = True class StreamEndStreamedChatResponse(_StreamEndStreamedChatResponse): - impacts: Optional[Impacts] = None + impacts: Optional[ImpactsOutput] = None class Config: arbitrary_types_allowed = True diff --git a/ecologits/tracers/huggingface_tracer.py b/ecologits/tracers/huggingface_tracer.py index bafd45fe..eaad94e5 100644 --- a/ecologits/tracers/huggingface_tracer.py +++ b/ecologits/tracers/huggingface_tracer.py @@ -10,20 +10,19 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "huggingface_hub" @dataclass class ChatCompletionOutput(_ChatCompletionOutput): - impacts: Impacts + impacts: ImpactsOutput @dataclass class ChatCompletionStreamOutput(_ChatCompletionStreamOutput): - impacts: Impacts + impacts: ImpactsOutput def huggingface_chat_wrapper( diff --git a/ecologits/tracers/litellm_tracer.py b/ecologits/tracers/litellm_tracer.py index 6723b0d4..03d7593d 100644 --- a/ecologits/tracers/litellm_tracer.py +++ b/ecologits/tracers/litellm_tracer.py @@ -9,17 +9,16 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts from ecologits.model_repository import models -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts class ChatCompletion(ModelResponse): - impacts: Impacts + impacts: ImpactsOutput class ChatCompletionChunk(ModelResponse): - impacts: Impacts + impacts: ImpactsOutput _model_choices = [f"{m.provider.value}/{m.name}" for m in models.list_models()] diff --git a/ecologits/tracers/mistralai_tracer_v0.py b/ecologits/tracers/mistralai_tracer_v0.py index f849bde9..e1020ccb 100644 --- a/ecologits/tracers/mistralai_tracer_v0.py +++ b/ecologits/tracers/mistralai_tracer_v0.py @@ -13,18 +13,17 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "mistralai" class ChatCompletionResponse(_ChatCompletionResponse): - impacts: Impacts + impacts: ImpactsOutput class ChatCompletionStreamResponse(_ChatCompletionStreamResponse): - impacts: Impacts + impacts: ImpactsOutput diff --git a/ecologits/tracers/mistralai_tracer_v1.py b/ecologits/tracers/mistralai_tracer_v1.py index 0595c5fb..80b58ae4 100644 --- a/ecologits/tracers/mistralai_tracer_v1.py +++ b/ecologits/tracers/mistralai_tracer_v1.py @@ -9,18 +9,17 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "mistralai" class ChatCompletionResponse(_ChatCompletionResponse): - impacts: Impacts + impacts: ImpactsOutput class CompletionChunk(_CompletionChunk): - impacts: Impacts + impacts: ImpactsOutput diff --git a/ecologits/tracers/openai_tracer.py b/ecologits/tracers/openai_tracer.py index c43645bb..6bda35aa 100644 --- a/ecologits/tracers/openai_tracer.py +++ b/ecologits/tracers/openai_tracer.py @@ -8,18 +8,17 @@ from wrapt import wrap_function_wrapper # type: ignore[import-untyped] from ecologits._ecologits import EcoLogits -from ecologits.impacts import Impacts -from ecologits.tracers.utils import llm_impacts +from ecologits.tracers.utils import ImpactsOutput, llm_impacts PROVIDER = "openai" class ChatCompletion(_ChatCompletion): - impacts: Impacts + impacts: ImpactsOutput class ChatCompletionChunk(_ChatCompletionChunk): - impacts: Impacts + impacts: ImpactsOutput def openai_chat_wrapper( diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index f9419dbb..494b0eec 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -1,15 +1,55 @@ from typing import Optional -from ecologits.alerts import ModelNotRegisteredError, ZoneNotRegisteredError +from pydantic import BaseModel + from ecologits.electricity_mix_repository import electricity_mixes from ecologits.impacts.llm import compute_llm_impacts -from ecologits.impacts.modeling import Impacts +from ecologits.impacts.modeling import GWP, PE, ADPe, Embodied, Energy, Usage from ecologits.log import logger from ecologits.model_repository import ParametersMoE, models +from ecologits.status_messages import ErrorMessage, ModelNotRegisteredError, WarningMessage, ZoneNotRegisteredError + + +class ImpactsOutput(BaseModel): + """ + Impacts output data model. + + Attributes: + energy: Total energy consumption + gwp: Total Global Warming Potential (GWP) impact + adpe: Total Abiotic Depletion Potential for Elements (ADPe) impact + pe: Total Primary Energy (PE) impact + usage: Impacts for the usage phase + embodied: Impacts for the embodied phase + warnings: List of warnings + errors: List of errors + """ + energy: Optional[Energy] = None + gwp: Optional[GWP] = None + adpe: Optional[ADPe] = None + pe: Optional[PE] = None + usage: Optional[Usage] = None + embodied: Optional[Embodied] = None + warnings: Optional[list[WarningMessage]] = None + errors: Optional[list[ErrorMessage]] = None + + @property + def has_warnings(self) -> bool: + return isinstance(self.warnings, list) and len(self.warnings) > 0 + + @property + def has_errors(self) -> bool: + return isinstance(self.errors, list) and len(self.errors) > 0 + def add_warning(self, warning: WarningMessage) -> None: + if self.warnings is None: + self.warnings = [] + self.warnings.append(warning) -def _avg(value_range: tuple) -> float: - return sum(value_range) / len(value_range) + def add_errors(self, error: ErrorMessage) -> None: + if self.errors is None: + self.errors = [] + self.errors.append(error) def llm_impacts( @@ -18,7 +58,7 @@ def llm_impacts( output_token_count: int, request_latency: float, electricity_mix_zone: str = "WOR", -) -> Optional[Impacts]: +) -> ImpactsOutput: """ High-level function to compute the impacts of an LLM generation request. @@ -36,7 +76,7 @@ def llm_impacts( model = models.find_model(provider=provider, model_name=model_name) if model is None: logger.debug(f"Could not find model `{model_name}` for {provider} provider.") - return Impacts(errors=[ModelNotRegisteredError()]) + return ImpactsOutput(errors=[ModelNotRegisteredError()]) if isinstance(model.architecture.parameters, ParametersMoE): model_total_params = model.architecture.parameters.total @@ -48,7 +88,7 @@ def llm_impacts( electricity_mix = electricity_mixes.find_electricity_mix(zone=electricity_mix_zone) if electricity_mix is None: logger.debug(f"Could not find electricity mix `{electricity_mix_zone}` in the ADEME database") - return Impacts(errors=[ZoneNotRegisteredError()]) + return ImpactsOutput(errors=[ZoneNotRegisteredError()]) if_electricity_mix_adpe=electricity_mix.adpe if_electricity_mix_pe=electricity_mix.pe if_electricity_mix_gwp=electricity_mix.gwp @@ -61,6 +101,7 @@ def llm_impacts( if_electricity_mix_pe=if_electricity_mix_pe, if_electricity_mix_gwp=if_electricity_mix_gwp, ) + impacts = ImpactsOutput.model_validate(impacts.model_dump()) if model.has_warnings: for w in model.warnings: diff --git a/mkdocs.yml b/mkdocs.yml index 84dd89a7..d2b336fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,7 +13,7 @@ nav: - 'Tutorial': - 'Introduction': tutorial/index.md - 'Environmental Impacts': tutorial/impacts.md - - 'Warnings and Errors': tutorial/alerts.md + - 'Warnings and Errors': tutorial/warnings_and_errors.md - 'Supported providers': tutorial/providers.md - 'Providers': - 'Anthropic': tutorial/providers/anthropic.md diff --git a/tests/test_alerts.py b/tests/test_status_messages.py similarity index 96% rename from tests/test_alerts.py rename to tests/test_status_messages.py index 8219d4d4..02ea9f4c 100644 --- a/tests/test_alerts.py +++ b/tests/test_status_messages.py @@ -1,7 +1,5 @@ -import pytest - from ecologits.tracers.utils import llm_impacts -from ecologits.alerts import ( +from ecologits.status_messages import ( ModelArchNotReleasedWarning, ModelArchMultimodalWarning, ModelNotRegisteredError, From 4d043fbb30f51491aa86bd09038e6ab2d8882963 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Sat, 7 Dec 2024 15:53:31 +0100 Subject: [PATCH 06/13] docs: update references for impacts object --- docs/tutorial/impacts.md | 17 +++++++++++------ ecologits/status_messages.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/tutorial/impacts.md b/docs/tutorial/impacts.md index 30ff6d7f..fe14de84 100644 --- a/docs/tutorial/impacts.md +++ b/docs/tutorial/impacts.md @@ -1,13 +1,14 @@ # Environmental Impacts -Environmental impacts are reported for each request in the [`Impacts`][impacts.modeling.Impacts] pydantic model and features multiple [criteria](#criteria) such as the [energy](#energy) and [global warming potential](#global-warming-potential-gwp) per phase ([usage](#usage) or [embodied](#embodied)) as well as the total impacts. +Environmental impacts are reported for each request in the [`ImpactsOutput`][tracers.utils.ImpactsOutput] pydantic model and features multiple [criteria](#criteria) such as the [energy](#energy) and [global warming potential](#global-warming-potential-gwp) per phase ([usage](#usage) or [embodied](#embodied)) as well as the total impacts. To learn more on how we estimate the environmental impacts and what are our hypotheses go to the [methodology](../methodology/index.md) section. ```python title="Structure of Impacts model" -from ecologits.impacts.modeling import * +from ecologits.tracers.utils import ImpactsOutput +from ecologits.impacts.modeling import ADPe, Embodied, Energy, GWP, PE, Usage -Impacts( +ImpactsOutput( energy=Energy(), # (1)! gwp=GWP(), adpe=ADPe(), @@ -22,13 +23,17 @@ Impacts( gwp=GWP(), adpe=ADPe(), pe=PE(), - ) + ), + warnings=None, # (4)! + errors=None ) ``` 1. Total impacts for all phases. 2. Usage impacts for the electricity consumption impacts. Note that the energy is equal to the "total" energy impact. 3. Embodied impacts for resource extract, manufacturing and transportation of hardware components allocated to the request. +4. List of [`WarningMessage`][status_messages.WarningMessage] and [`ErrorMessage`][status_messages.ErrorMessage]. + You can extract an impact with: @@ -43,10 +48,10 @@ Or you could get **value range** impact instead: ```python >>> response.impacts.usage.gwp.value -Range(min=0.16, max=0.48) # Expressed in kgCO2eq (1) +RangeValue(min=0.16, max=0.48) # Expressed in kgCO2eq (1) ``` -1. [`Range`][impacts.modeling.Range] are used to define intervals. +1. [`RangeValue`][utils.range_value.RangeValue] are used to define intervals. ## Criteria diff --git a/ecologits/status_messages.py b/ecologits/status_messages.py index 1e90152c..7caf40ed 100644 --- a/ecologits/status_messages.py +++ b/ecologits/status_messages.py @@ -23,6 +23,13 @@ def from_code(cls, code: str) -> type["_StatusMessage"]: class WarningMessage(_StatusMessage): + """ + Warning message. + + Attributes: + code: Warning code. + message: Warning message. + """ @classmethod def from_code(cls, code: str) -> "WarningMessage": @@ -33,6 +40,13 @@ def from_code(cls, code: str) -> "WarningMessage": class ErrorMessage(_StatusMessage): + """ + Error message. + + Attributes: + code: Error code. + message: Error message. + """ @classmethod def from_code(cls, code: str) -> "ErrorMessage": From 4bd3e4ebafc27efe2ccf8c27fadabb714526ac14 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Sat, 7 Dec 2024 15:59:03 +0100 Subject: [PATCH 07/13] docs: update reference for ImpactsOutput object --- docs/tutorial/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial/index.md b/docs/tutorial/index.md index 4232ba8d..d8b0cbf2 100644 --- a/docs/tutorial/index.md +++ b/docs/tutorial/index.md @@ -2,7 +2,7 @@ The :seedling: **EcoLogits** library tracks the energy consumption and environmental impacts of generative AI models accessed through APIs and their official client libraries. -It achieves this by **patching the Python client libraries**, ensuring that each API request is wrapped with an impact calculation function. This function computes the **environmental impact based on several request features**, such as the **chosen model**, the **number of tokens generated**, and the **request's latency**. The resulting data is then encapsulated in an `Impacts` object, which is added to the response, containing the environmental impacts for a specific request. +It achieves this by **patching the Python client libraries**, ensuring that each API request is wrapped with an impact calculation function. This function computes the **environmental impact based on several request features**, such as the **chosen model**, the **number of tokens generated**, and the **request's latency**. The resulting data is then encapsulated in an [`ImpactsOutput`][tracers.utils.ImpactsOutput] object, which is added to the response, containing the environmental impacts for a specific request.
From 0a0cc531408380b288dc970682f232863224fc79 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Sat, 7 Dec 2024 16:02:12 +0100 Subject: [PATCH 08/13] style: remove newline in status messages --- ecologits/status_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecologits/status_messages.py b/ecologits/status_messages.py index 7caf40ed..95f1ecef 100644 --- a/ecologits/status_messages.py +++ b/ecologits/status_messages.py @@ -15,7 +15,7 @@ class _StatusMessage(BaseModel): message: str def __str__(self) -> str: - return f"{self.message}\n\nFor further information visit {STATUS_DOCS_URL.format(code=self.code)}" + return f"{self.message} For further information visit {STATUS_DOCS_URL.format(code=self.code)}" @classmethod def from_code(cls, code: str) -> type["_StatusMessage"]: From a1e4827fb615c70d0befc005db1e52512c5fd66e Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Mon, 9 Dec 2024 14:08:56 +0100 Subject: [PATCH 09/13] feat: add logger methods to log a message only once --- ecologits/log.py | 29 +++++++++++++++++++++++++++++ ecologits/tracers/utils.py | 13 +++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/ecologits/log.py b/ecologits/log.py index eea436a3..edf05a3f 100644 --- a/ecologits/log.py +++ b/ecologits/log.py @@ -1,3 +1,32 @@ import logging + +class EcoLogitsLogger(logging.Logger): + + def __init__(self, name, level=logging.NOTSET): + super().__init__(name, level) + self.__once_messages = set() + + def _log_once(self, level, msg, *args, **kwargs): + if msg not in self.__once_messages: + self.__once_messages.add(msg) + self.log(level, msg, *args, **kwargs) + + def debug_once(self, msg, *args, **kwargs): + self._log_once(logging.DEBUG, msg, *args, **kwargs) + + def info_once(self, msg, *args, **kwargs): + self._log_once(logging.INFO, msg, *args, **kwargs) + + def warning_once(self, msg, *args, **kwargs): + self._log_once(logging.WARNING, msg, *args, **kwargs) + + def error_once(self, msg, *args, **kwargs): + self._log_once(logging.ERROR, msg, *args, **kwargs) + + def critical_once(self, msg, *args, **kwargs): + self._log_once(logging.CRITICAL, msg, *args, **kwargs) + + +logging.setLoggerClass(EcoLogitsLogger) logger = logging.getLogger(__name__) diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index 494b0eec..06b77e60 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -75,8 +75,9 @@ def llm_impacts( model = models.find_model(provider=provider, model_name=model_name) if model is None: - logger.debug(f"Could not find model `{model_name}` for {provider} provider.") - return ImpactsOutput(errors=[ModelNotRegisteredError()]) + error = ModelNotRegisteredError(message=f"Could not find model `{model_name}` for {provider} provider.") + logger.warning_once(str(error)) + return ImpactsOutput(errors=[error]) if isinstance(model.architecture.parameters, ParametersMoE): model_total_params = model.architecture.parameters.total @@ -87,8 +88,11 @@ def llm_impacts( electricity_mix = electricity_mixes.find_electricity_mix(zone=electricity_mix_zone) if electricity_mix is None: - logger.debug(f"Could not find electricity mix `{electricity_mix_zone}` in the ADEME database") - return ImpactsOutput(errors=[ZoneNotRegisteredError()]) + error = ModelNotRegisteredError(message=f"Could not find electricity mix `{electricity_mix_zone}` in the " + f"ADEME database.") + logger.warning_once(str(error)) + return ImpactsOutput(errors=[error]) + if_electricity_mix_adpe=electricity_mix.adpe if_electricity_mix_pe=electricity_mix.pe if_electricity_mix_gwp=electricity_mix.gwp @@ -105,6 +109,7 @@ def llm_impacts( if model.has_warnings: for w in model.warnings: + logger.warning_once(str(w)) impacts.add_warning(w) return impacts From 1bf31bdf9a263b751ca0a7079a31d426a42894da Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Fri, 13 Dec 2024 16:23:54 +0100 Subject: [PATCH 10/13] fix: change error type for unknown zone --- ecologits/tracers/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecologits/tracers/utils.py b/ecologits/tracers/utils.py index 06b77e60..252e128f 100644 --- a/ecologits/tracers/utils.py +++ b/ecologits/tracers/utils.py @@ -88,8 +88,7 @@ def llm_impacts( electricity_mix = electricity_mixes.find_electricity_mix(zone=electricity_mix_zone) if electricity_mix is None: - error = ModelNotRegisteredError(message=f"Could not find electricity mix `{electricity_mix_zone}` in the " - f"ADEME database.") + error = ZoneNotRegisteredError(message=f"Could not find electricity mix for `{electricity_mix_zone}` zone.") logger.warning_once(str(error)) return ImpactsOutput(errors=[error]) From 555a18d38caa614303154d7f402eb0f04c5f123e Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Fri, 13 Dec 2024 16:35:02 +0100 Subject: [PATCH 11/13] fix: add types to custom logger --- ecologits/log.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ecologits/log.py b/ecologits/log.py index edf05a3f..bd440bee 100644 --- a/ecologits/log.py +++ b/ecologits/log.py @@ -1,32 +1,33 @@ import logging +from typing import Any, Union, cast class EcoLogitsLogger(logging.Logger): - def __init__(self, name, level=logging.NOTSET): + def __init__(self, name: str, level: Union[int, str] = logging.NOTSET) -> None: super().__init__(name, level) - self.__once_messages = set() + self.__once_messages: set[str] = set() - def _log_once(self, level, msg, *args, **kwargs): + def _log_once(self, level: int, msg: str, *args: Any, **kwargs: Any) -> None: if msg not in self.__once_messages: self.__once_messages.add(msg) self.log(level, msg, *args, **kwargs) - def debug_once(self, msg, *args, **kwargs): + def debug_once(self, msg: str, *args: Any, **kwargs: Any) -> None: self._log_once(logging.DEBUG, msg, *args, **kwargs) - def info_once(self, msg, *args, **kwargs): + def info_once(self, msg: str, *args: Any, **kwargs: Any) -> None: self._log_once(logging.INFO, msg, *args, **kwargs) - def warning_once(self, msg, *args, **kwargs): + def warning_once(self, msg: str, *args: Any, **kwargs: Any) -> None: self._log_once(logging.WARNING, msg, *args, **kwargs) - def error_once(self, msg, *args, **kwargs): + def error_once(self, msg: str, *args: Any, **kwargs: Any) -> None: self._log_once(logging.ERROR, msg, *args, **kwargs) - def critical_once(self, msg, *args, **kwargs): + def critical_once(self, msg: str, *args: Any, **kwargs: Any) -> None: self._log_once(logging.CRITICAL, msg, *args, **kwargs) logging.setLoggerClass(EcoLogitsLogger) -logger = logging.getLogger(__name__) +logger: EcoLogitsLogger = cast(EcoLogitsLogger, logging.getLogger(__name__)) From fcd0c05572a34e3104c9dd7dc8c27e7fc579df7c Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Fri, 13 Dec 2024 16:55:14 +0100 Subject: [PATCH 12/13] fix: use "ecologits" as logger name --- ecologits/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecologits/log.py b/ecologits/log.py index bd440bee..51468f5d 100644 --- a/ecologits/log.py +++ b/ecologits/log.py @@ -30,4 +30,4 @@ def critical_once(self, msg: str, *args: Any, **kwargs: Any) -> None: logging.setLoggerClass(EcoLogitsLogger) -logger: EcoLogitsLogger = cast(EcoLogitsLogger, logging.getLogger(__name__)) +logger: EcoLogitsLogger = cast(EcoLogitsLogger, logging.getLogger("ecologits")) From 636ac8b0365fc98ae9d56d6bd347c05160cb4715 Mon Sep 17 00:00:00 2001 From: Samuel Rince Date: Fri, 13 Dec 2024 16:55:37 +0100 Subject: [PATCH 13/13] tests: add logging and logging once tests --- tests/test_logger.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/test_logger.py diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 00000000..a7d93131 --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,35 @@ +import logging + +import pytest + +from ecologits.log import logger + + +@pytest.mark.parametrize("logging_func", [ + logger.debug, + logger.info, + logger.warning, + logger.error, + logger.critical +]) +def test_logging(caplog, logging_func): + with caplog.at_level(logging.DEBUG, logger="ecologits"): + logging_func("test") + assert "test" in caplog.text + + +@pytest.mark.parametrize("logging_func", [ + logger.debug_once, + logger.info_once, + logger.warning_once, + logger.error_once, + logger.critical_once +]) +def test_logging_once(caplog, logging_func): + with caplog.at_level(logging.DEBUG, logger="ecologits"): + logging_func(f"test({logging_func.__name__})") + logging_func(f"test({logging_func.__name__})") # This shouldn't be logged + logging_func(f"test2({logging_func.__name__})") + assert len(caplog.records) == 2 + assert "test" in caplog.text + assert "test2" in caplog.text