Add support for rate limiting open ai calls

docs/index.md

@@ -31,7 +31,7 @@ In order to run a local demo service based on the IEKit:
 2. Fetch the IEKit.
 
     ```bash
-    autonomy fetch valory/impact_evaluator:0.1.0:bafybeibr2sxcwdbcwjeilaehr7vl2qoqoyigm2italdtiyvla4vq5r76ky --service
+    autonomy fetch valory/impact_evaluator:0.1.0:bafybeiewglbmk5dlqp4kvujry5pgwc34gwnjltaidd4tlb4hoffrd3ahru --service
     ```
 
 3. Build the Docker image of the service agents

packages/packages.json

@@ -1,13 +1,13 @@
 {
     "dev": {
-        "agent/valory/impact_evaluator/0.1.0": "bafybeiaa2f7rmpnzwaltyoryuyv3rylt6agrewgmsidwdgpwd7iyituazi",
+        "agent/valory/impact_evaluator/0.1.0": "bafybeibdhdgmges4ieepj7t37kaca47lsacga3cx64pdcogaogkro34wg4",
         "contract/valory/dynamic_contribution/0.1.0": "bafybeie76ynpueo3hh2ujzcfgpqcsbyqwa4pdcc6g44a4he6bueyb4tqiy",
-        "service/valory/impact_evaluator/0.1.0": "bafybeibr2sxcwdbcwjeilaehr7vl2qoqoyigm2italdtiyvla4vq5r76ky",
+        "service/valory/impact_evaluator/0.1.0": "bafybeiewglbmk5dlqp4kvujry5pgwc34gwnjltaidd4tlb4hoffrd3ahru",
         "skill/valory/dynamic_nft_abci/0.1.0": "bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq",
-        "skill/valory/twitter_scoring_abci/0.1.0": "bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju",
+        "skill/valory/twitter_scoring_abci/0.1.0": "bafybeidw7pry4s3eeds5mjt2yd7b2bznj2koqib2wejpwqux2lt3e3qo3e",
         "skill/valory/ceramic_read_abci/0.1.0": "bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia",
         "skill/valory/ceramic_write_abci/0.1.0": "bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay",
-        "skill/valory/impact_evaluator_abci/0.1.0": "bafybeichibvsp2juk72evfxostkr7zsvejekepuekh62qr5vvlmn6tvcnu",
+        "skill/valory/impact_evaluator_abci/0.1.0": "bafybeid42ll3rb247ikp3aia53q37ltnvununr76q7rfexnosnimwwerxy",
         "skill/valory/generic_scoring_abci/0.1.0": "bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq",
         "protocol/valory/twitter/0.1.0": "bafybeib4eyf7qbs7kdntqzhwqsaaj4o2mzcokcztaza6qgwt7sbxgkqu2m",
         "protocol/valory/llm/1.0.0": "bafybeigqybmg75vsxexmp57hkms7lkp7iwpf54r7wpygizxryvrhfqqpb4",

packages/valory/agents/impact_evaluator/aea-config.yaml

@@ -42,9 +42,9 @@ protocols:
 skills:
 - valory/abstract_abci:0.1.0:bafybeicg7dv7cff34nv2k2z47c4yp4kddsxp3wozonzow6tnvfvwndz3cy
 - valory/abstract_round_abci:0.1.0:bafybeigxjcci53vwytymzlhr37436yvenh7jup4astrn7dgyixo24aq2pq
-- valory/impact_evaluator_abci:0.1.0:bafybeichibvsp2juk72evfxostkr7zsvejekepuekh62qr5vvlmn6tvcnu
+- valory/impact_evaluator_abci:0.1.0:bafybeid42ll3rb247ikp3aia53q37ltnvununr76q7rfexnosnimwwerxy
 - valory/generic_scoring_abci:0.1.0:bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq
-- valory/twitter_scoring_abci:0.1.0:bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju
+- valory/twitter_scoring_abci:0.1.0:bafybeidw7pry4s3eeds5mjt2yd7b2bznj2koqib2wejpwqux2lt3e3qo3e
 - valory/ceramic_read_abci:0.1.0:bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia
 - valory/ceramic_write_abci:0.1.0:bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay
 - valory/dynamic_nft_abci:0.1.0:bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq
@@ -212,6 +212,8 @@ models:
       twitter_max_pages: 10
       twitter_search_endpoint: ${str:2/tweets/search/recent?}
       twitter_search_args: ${str:query=%23olas&tweet.fields=author_id,created_at&user.fields=name&expansions=author_id&max_results=100&since_id={since_id}}
+      openai_call_window_size: ${float:3600.0}
+      openai_calls_allowed_in_window: ${int:100}
       tx_timeout: 10.0
       use_termination: ${bool:false}
       validate_timeout: 1205

packages/valory/services/impact_evaluator/service.yaml

@@ -8,7 +8,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeign56hilwuoa6bgos3uqabss4gew4vadkik7vhj3ucpqw6nxtqtpe
 fingerprint_ignore_patterns: []
-agent: valory/impact_evaluator:0.1.0:bafybeiaa2f7rmpnzwaltyoryuyv3rylt6agrewgmsidwdgpwd7iyituazi
+agent: valory/impact_evaluator:0.1.0:bafybeibdhdgmges4ieepj7t37kaca47lsacga3cx64pdcogaogkro34wg4
 number_of_agents: 4
 deployment:
   agent:
@@ -112,6 +112,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 1:
   models:
     benchmark_tool:
@@ -176,6 +178,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 2:
   models:
     benchmark_tool:
@@ -240,6 +244,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 3:
   models:
     benchmark_tool:
@@ -308,6 +314,8 @@ extra:
         history_check_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 ---
 public_id: valory/ledger:0.19.0
 type: connection

packages/valory/skills/impact_evaluator_abci/composition.py

@@ -51,8 +51,9 @@
     DecisionMakingAbci.FinishedDecisionMakingReadManualPointsRound: CeramicReadAbci.StreamReadRound,
     DecisionMakingAbci.FinishedDecisionMakingScoreRound: GenericScoringAbci.GenericScoringRound,
     DecisionMakingAbci.FinishedDecisionMakingDoneRound: ResetAndPauseAbci.ResetAndPauseRound,
-    GenericScoringAbci.FinishedGenericScoringRound: TwitterScoringAbci.TwitterCollectionRound,
+    GenericScoringAbci.FinishedGenericScoringRound: TwitterScoringAbci.OpenAICallCheckRound,
     TwitterScoringAbci.FinishedTwitterScoringRound: DynamicNFTAbci.TokenTrackRound,
+    TwitterScoringAbci.FinishedTwitterScoringWithAPIErrorRound: DynamicNFTAbci.TokenTrackRound,
     DynamicNFTAbci.FinishedTokenTrackRound: DecisionMakingAbci.DecisionMakingRound,
     LLMAbciApp.FinishedLLMRound: DecisionMakingAbci.DecisionMakingRound,
     TwitterWriteAbciApp.FinishedTwitterWriteRound: DecisionMakingAbci.DecisionMakingRound,

packages/valory/skills/impact_evaluator_abci/fsm_specification.yaml

@@ -37,6 +37,7 @@ states:
 - LLMRandomnessRound
 - LLMRound
 - LLMSelectKeeperRound
+- OpenAICallCheckRound
 - RandomnessRound
 - RandomnessTwitterRound
 - RegistrationRound
@@ -69,7 +70,7 @@ transition_func:
     (DecisionMakingRound, SCORE): GenericScoringRound
     (DecisionMakingRound, UPDATE_CENTAURS): RandomnessRound
     (DecisionMakingRound, WRITE_CONTRIBUTE_DB): RandomnessRound
-    (GenericScoringRound, DONE): TwitterCollectionRound
+    (GenericScoringRound, DONE): OpenAICallCheckRound
     (GenericScoringRound, NO_MAJORITY): GenericScoringRound
     (GenericScoringRound, ROUND_TIMEOUT): GenericScoringRound
     (LLMRandomnessRound, DONE): LLMSelectKeeperRound
@@ -82,6 +83,9 @@ transition_func:
     (LLMSelectKeeperRound, DONE): LLMRound
     (LLMSelectKeeperRound, NO_MAJORITY): LLMRandomnessRound
     (LLMSelectKeeperRound, ROUND_TIMEOUT): LLMRandomnessRound
+    (OpenAICallCheckRound, API_ERROR): TokenTrackRound
+    (OpenAICallCheckRound, DONE): TwitterCollectionRound
+    (OpenAICallCheckRound, NO_MAJORITY): OpenAICallCheckRound
     (RandomnessRound, DONE): SelectKeeperRound
     (RandomnessRound, NO_MAJORITY): RandomnessRound
     (RandomnessRound, ROUND_TIMEOUT): RandomnessRound

packages/valory/skills/impact_evaluator_abci/models.py

@@ -19,6 +19,10 @@
 
 """This module contains the shared state for the abci skill of ImpactEvaluatorSkillAbciApp."""
 
+from typing import Any, cast
+
+from aea.skills.base import SkillContext
+
 from packages.valory.skills.abstract_round_abci.models import ApiSpecs
 from packages.valory.skills.abstract_round_abci.models import (
     BenchmarkTool as BaseBenchmarkTool,
@@ -58,6 +62,7 @@
 from packages.valory.skills.llm_abci.rounds import Event as LLMEvent
 from packages.valory.skills.reset_pause_abci.rounds import Event as ResetPauseEvent
 from packages.valory.skills.termination_abci.models import TerminationParams
+from packages.valory.skills.twitter_scoring_abci.models import OpenAICalls
 from packages.valory.skills.twitter_scoring_abci.models import (
     Params as TwitterScoringAbciParams,
 )
@@ -90,6 +95,18 @@ class SharedState(BaseSharedState):
 
     abci_app_cls = ImpactEvaluatorSkillAbciApp
 
+    def __init__(self, *args: Any, skill_context: SkillContext, **kwargs: Any) -> None:
+        """Initialize object."""
+        super().__init__(*args, skill_context=skill_context, **kwargs)
+        self.openai_calls = OpenAICalls(
+            openai_call_window_size=cast(
+                "Params", self.context.params
+            ).openai_call_window_size,
+            openai_calls_allowed_in_window=cast(
+                "Params", self.context.params
+            ).openai_calls_allowed_in_window,
+        )
+
     def setup(self) -> None:
         """Set up."""
         super().setup()

packages/valory/skills/impact_evaluator_abci/skill.yaml

@@ -8,11 +8,11 @@ aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
   __init__.py: bafybeigmhxenrqb2ysjhcm2au2lang4abyny6irkzqqg4dtclz6net6zpy
   behaviours.py: bafybeifhoufesr4r4re65mxjadqasjymvz6tkc25xn5yyn473yunhjiufe
-  composition.py: bafybeifvic3nzwujprattfcyxzxzp7g76x4ye24tte5p6rpvv6oe7sjzgm
+  composition.py: bafybeicjgndv2v43wclaoldmhpgugajsutqjfbhwbw67cmrli7z3eqpxne
   dialogues.py: bafybeigjknz4qqynbsltjje46gidg4rftsqw6ybjwegz24wetmycutpzh4
-  fsm_specification.yaml: bafybeigfsjmxb7rbaup5ozfw2r54zh3qzlc3aeuozdsnegavbwf37bhpjy
+  fsm_specification.yaml: bafybeibqudgc2r4n4maicfi3mchfqaznw6evy4iyg7keuezkub7dl6zlvq
   handlers.py: bafybeidkli6fphcmdgwsys4lkyf3fx6fbawet4nt2pnixfypzijhg6b3ze
-  models.py: bafybeifhcshu4iwqvc2fz3auu5ngfykkhvqnroeug52ilrwwc4kowkeg5a
+  models.py: bafybeihk7nrlzak42eygucwenw5widne5jcz4vhcu6nkvs443mdpihemnm
   tests/__init__.py: bafybeievwzwojvq4aofk5kjpf4jzygfes7ew6s6svc6b6frktjnt3sicce
   tests/test_behaviours.py: bafybeifkazsevd6vsfaapulouxepez3rl24y3rxgja5zhmj7s323zdjlmq
   tests/test_dialogues.py: bafybeieaos2byphju6i6xvytppqqcuqqvnpilnflsy73l3wqazzjttbg7m
@@ -26,7 +26,7 @@ skills:
 - valory/abstract_round_abci:0.1.0:bafybeigxjcci53vwytymzlhr37436yvenh7jup4astrn7dgyixo24aq2pq
 - valory/ceramic_read_abci:0.1.0:bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia
 - valory/generic_scoring_abci:0.1.0:bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq
-- valory/twitter_scoring_abci:0.1.0:bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju
+- valory/twitter_scoring_abci:0.1.0:bafybeidw7pry4s3eeds5mjt2yd7b2bznj2koqib2wejpwqux2lt3e3qo3e
 - valory/ceramic_write_abci:0.1.0:bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay
 - valory/dynamic_nft_abci:0.1.0:bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq
 - valory/registration_abci:0.1.0:bafybeibc4kczqbh23sc6tufrzn3axmhp3vjav7fa3u6cnpvolrbbc2fd7i
@@ -169,6 +169,8 @@ models:
       twitter_max_pages: 10
       twitter_search_endpoint: 2/tweets/search/recent?
       twitter_search_args: query=%23olas&tweet.fields=author_id,created_at&user.fields=name&expansions=author_id&max_results=100&since_id={since_id}
+      openai_call_window_size: 3600.0
+      openai_calls_allowed_in_window: 100
       max_points_per_period: 5000
       tx_timeout: 10.0
       whitelist_api_key: null

packages/valory/skills/twitter_scoring_abci/behaviours.py

@@ -42,15 +42,21 @@
     LlmDialogue,
     LlmDialogues,
 )
-from packages.valory.skills.twitter_scoring_abci.models import Params, SharedState
+from packages.valory.skills.twitter_scoring_abci.models import (
+    OpenAICalls,
+    Params,
+    SharedState,
+)
 from packages.valory.skills.twitter_scoring_abci.payloads import (
     DBUpdatePayload,
+    OpenAICallCheckPayload,
     TweetEvaluationPayload,
     TwitterCollectionPayload,
 )
 from packages.valory.skills.twitter_scoring_abci.prompts import tweet_evaluation_prompt
 from packages.valory.skills.twitter_scoring_abci.rounds import (
     DBUpdateRound,
+    OpenAICallCheckRound,
     SynchronizedData,
     TweetEvaluationRound,
     TwitterCollectionRound,
@@ -78,6 +84,39 @@ def params(self) -> Params:
         """Return the params."""
         return cast(Params, super().params)
 
+    @property
+    def openai_calls(self) -> OpenAICalls:
+        """Return the params."""
+        return cast(OpenAICalls, self.shared_state.openai_calls)
+
+
+class OpenAICallCheckBehaviour(TwitterScoringBaseBehaviour):
+    """TwitterCollectionBehaviour"""
+
+    matching_round: Type[AbstractRound] = OpenAICallCheckRound
+
+    def async_act(self) -> Generator:
+        """Do the act, supporting asynchronous execution."""
+        with self.context.benchmark_tool.measure(self.behaviour_id).local():
+            current_time = cast(
+                SharedState, self.context.state
+            ).round_sequence.last_round_transition_timestamp.timestamp()
+            # Reset the window if the window expired before checking
+            self.openai_calls.reset(current_time=current_time)
+            if self.openai_calls.max_tweets_reached():
+                content = None
+            else:
+                content = OpenAICallCheckRound.CALLS_REMAINING
+        with self.context.benchmark_tool.measure(self.behaviour_id).consensus():
+            yield from self.send_a2a_transaction(
+                payload=OpenAICallCheckPayload(
+                    sender=self.context.agent_address,
+                    content=content,
+                )
+            )
+            yield from self.wait_until_round_end()
+        self.set_done()
+
 
 class TwitterCollectionBehaviour(TwitterScoringBaseBehaviour):
     """TwitterCollectionBehaviour"""
@@ -389,7 +428,7 @@ def evaluate_tweet(self, text: str) -> Generator[None, None, int]:
             request_llm_message, llm_dialogue
         )
         data = llm_response_message.value
-
+        self.openai_calls.increase_call_count()
         self.context.logger.info(f"Got tweet evaluation: {repr(data)}")
 
         points = DEFAULT_TWEET_POINTS
@@ -582,6 +621,7 @@ class TwitterScoringRoundBehaviour(AbstractRoundBehaviour):
     initial_behaviour_cls = TwitterCollectionBehaviour
     abci_app_cls = TwitterScoringAbciApp  # type: ignore
     behaviours: Set[Type[BaseBehaviour]] = [
+        OpenAICallCheckBehaviour,
         TwitterCollectionBehaviour,
         TweetEvaluationBehaviour,
         DBUpdateBehaviour,

packages/valory/skills/twitter_scoring_abci/fsm_specification.yaml

@@ -5,21 +5,27 @@ alphabet_in:
 - ROUND_TIMEOUT
 - SKIP
 - TWEET_EVALUATION_ROUND_TIMEOUT
-default_start_state: TwitterCollectionRound
+default_start_state: OpenAICallCheckRound
 final_states:
 - FinishedTwitterScoringRound
+- FinishedTwitterScoringWithAPIErrorRound
 label: TwitterScoringAbciApp
 start_states:
-- TwitterCollectionRound
+- OpenAICallCheckRound
 states:
 - DBUpdateRound
 - FinishedTwitterScoringRound
+- FinishedTwitterScoringWithAPIErrorRound
+- OpenAICallCheckRound
 - TweetEvaluationRound
 - TwitterCollectionRound
 transition_func:
     (DBUpdateRound, DONE): FinishedTwitterScoringRound
     (DBUpdateRound, NO_MAJORITY): DBUpdateRound
     (DBUpdateRound, ROUND_TIMEOUT): DBUpdateRound
+    (OpenAICallCheckRound, API_ERROR): FinishedTwitterScoringWithAPIErrorRound
+    (OpenAICallCheckRound, DONE): TwitterCollectionRound
+    (OpenAICallCheckRound, NO_MAJORITY): OpenAICallCheckRound
     (TweetEvaluationRound, DONE): DBUpdateRound
     (TweetEvaluationRound, TWEET_EVALUATION_ROUND_TIMEOUT): TweetEvaluationRound
     (TwitterCollectionRound, API_ERROR): TwitterCollectionRound

packages/valory/skills/twitter_scoring_abci/models.py

@@ -19,7 +19,10 @@
 
 """This module contains the shared state for the abci skill of TwitterScoringAbciApp."""
 
-from typing import Any
+from datetime import datetime
+from typing import Any, cast
+
+from aea.skills.base import SkillContext
 
 from packages.valory.skills.abstract_round_abci.models import BaseParams
 from packages.valory.skills.abstract_round_abci.models import (
@@ -37,6 +40,52 @@ class SharedState(BaseSharedState):
 
     abci_app_cls = TwitterScoringAbciApp
 
+    def __init__(self, *args: Any, skill_context: SkillContext, **kwargs: Any) -> None:
+        """Initialize object."""
+        super().__init__(*args, skill_context=skill_context, **kwargs)
+        self.openai_calls = OpenAICalls(
+            openai_call_window_size=cast(
+                "Params", self.context.params
+            ).openai_call_window_size,
+            openai_calls_allowed_in_window=cast(
+                "Params", self.context.params
+            ).openai_calls_allowed_in_window,
+        )
+
+
+class OpenAICalls:
+    """OpenAI call window."""
+
+    def __init__(
+        self,
+        openai_call_window_size: float,
+        openai_calls_allowed_in_window: int,
+    ) -> None:
+        """Initialize object."""
+        self._calls_made_in_window = 0
+        self._calls_allowed_in_window = openai_calls_allowed_in_window
+        self._call_window_size = openai_call_window_size
+        self._call_window_start = datetime.now().timestamp()
+
+    def increase_call_count(self) -> None:
+        """Increase call count."""
+        self._calls_made_in_window += 1
+
+    def has_window_expired(self, current_time: float) -> bool:
+        """Increase tweet count."""
+        return current_time > (self._call_window_start + self._call_window_size)
+
+    def max_calls_reached(self) -> bool:
+        """Increase tweet count."""
+        return self._calls_made_in_window >= self._calls_allowed_in_window
+
+    def reset(self, current_time: float) -> None:
+        """Reset the window if required.."""
+        if not self.has_window_expired(current_time=current_time):
+            return
+        self._tweets_made_in_window = 0
+        self._call_window_start = current_time
+
 
 class Params(BaseParams):
     """Parameters."""
@@ -60,6 +109,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self.tweet_evaluation_round_timeout = self._ensure(
             "tweet_evaluation_round_timeout", kwargs, float
         )
+        self.openai_call_window_size = self._ensure(
+            "openai_call_window_size", kwargs, float
+        )
+        self.openai_calls_allowed_in_window = self._ensure(
+            "openai_calls_allowed_in_window", kwargs, int
+        )
         super().__init__(*args, **kwargs)