Add support for rate limiting open ai calls

docs/index.md

@@ -31,7 +31,7 @@ In order to run a local demo service based on the IEKit:
 2. Fetch the IEKit.
 
     ```bash
-    autonomy fetch valory/impact_evaluator:0.1.0:bafybeibr2sxcwdbcwjeilaehr7vl2qoqoyigm2italdtiyvla4vq5r76ky --service
+    autonomy fetch valory/impact_evaluator:0.1.0:bafybeiery5vbtdvyvgwsmovyiazeawc3l72wevpsrpxx53ucjao75pizom --service
     ```
 
 3. Build the Docker image of the service agents

packages/packages.json

@@ -1,13 +1,13 @@
 {
     "dev": {
-        "agent/valory/impact_evaluator/0.1.0": "bafybeiaa2f7rmpnzwaltyoryuyv3rylt6agrewgmsidwdgpwd7iyituazi",
+        "agent/valory/impact_evaluator/0.1.0": "bafybeicc3axjlx2xt7da5mkpadyhexs2bzn3k77qcgywsin6pnt7u4l3va",
         "contract/valory/dynamic_contribution/0.1.0": "bafybeie76ynpueo3hh2ujzcfgpqcsbyqwa4pdcc6g44a4he6bueyb4tqiy",
-        "service/valory/impact_evaluator/0.1.0": "bafybeibr2sxcwdbcwjeilaehr7vl2qoqoyigm2italdtiyvla4vq5r76ky",
+        "service/valory/impact_evaluator/0.1.0": "bafybeiery5vbtdvyvgwsmovyiazeawc3l72wevpsrpxx53ucjao75pizom",
         "skill/valory/dynamic_nft_abci/0.1.0": "bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq",
-        "skill/valory/twitter_scoring_abci/0.1.0": "bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju",
+        "skill/valory/twitter_scoring_abci/0.1.0": "bafybeif63wgrs2ckqwfpfpfkrhrmhuykapqobmvry4a2dpfkpytnm4raam",
         "skill/valory/ceramic_read_abci/0.1.0": "bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia",
         "skill/valory/ceramic_write_abci/0.1.0": "bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay",
-        "skill/valory/impact_evaluator_abci/0.1.0": "bafybeichibvsp2juk72evfxostkr7zsvejekepuekh62qr5vvlmn6tvcnu",
+        "skill/valory/impact_evaluator_abci/0.1.0": "bafybeihxjxxc6fyze5fyuh6tlemy2ou6adpnr4ayoztqpuycm3k4lzlqxu",
         "skill/valory/generic_scoring_abci/0.1.0": "bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq",
         "protocol/valory/twitter/0.1.0": "bafybeib4eyf7qbs7kdntqzhwqsaaj4o2mzcokcztaza6qgwt7sbxgkqu2m",
         "protocol/valory/llm/1.0.0": "bafybeigqybmg75vsxexmp57hkms7lkp7iwpf54r7wpygizxryvrhfqqpb4",

packages/valory/agents/impact_evaluator/aea-config.yaml

@@ -42,9 +42,9 @@ protocols:
 skills:
 - valory/abstract_abci:0.1.0:bafybeicg7dv7cff34nv2k2z47c4yp4kddsxp3wozonzow6tnvfvwndz3cy
 - valory/abstract_round_abci:0.1.0:bafybeigxjcci53vwytymzlhr37436yvenh7jup4astrn7dgyixo24aq2pq
-- valory/impact_evaluator_abci:0.1.0:bafybeichibvsp2juk72evfxostkr7zsvejekepuekh62qr5vvlmn6tvcnu
+- valory/impact_evaluator_abci:0.1.0:bafybeihxjxxc6fyze5fyuh6tlemy2ou6adpnr4ayoztqpuycm3k4lzlqxu
 - valory/generic_scoring_abci:0.1.0:bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq
-- valory/twitter_scoring_abci:0.1.0:bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju
+- valory/twitter_scoring_abci:0.1.0:bafybeif63wgrs2ckqwfpfpfkrhrmhuykapqobmvry4a2dpfkpytnm4raam
 - valory/ceramic_read_abci:0.1.0:bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia
 - valory/ceramic_write_abci:0.1.0:bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay
 - valory/dynamic_nft_abci:0.1.0:bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq
@@ -212,6 +212,8 @@ models:
       twitter_max_pages: 10
       twitter_search_endpoint: ${str:2/tweets/search/recent?}
       twitter_search_args: ${str:query=%23olas&tweet.fields=author_id,created_at&user.fields=name&expansions=author_id&max_results=100&since_id={since_id}}
+      openai_call_window_size: ${float:3600.0}
+      openai_calls_allowed_in_window: ${int:100}
       tx_timeout: 10.0
       use_termination: ${bool:false}
       validate_timeout: 1205

packages/valory/services/impact_evaluator/service.yaml

@@ -8,7 +8,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeign56hilwuoa6bgos3uqabss4gew4vadkik7vhj3ucpqw6nxtqtpe
 fingerprint_ignore_patterns: []
-agent: valory/impact_evaluator:0.1.0:bafybeiaa2f7rmpnzwaltyoryuyv3rylt6agrewgmsidwdgpwd7iyituazi
+agent: valory/impact_evaluator:0.1.0:bafybeicc3axjlx2xt7da5mkpadyhexs2bzn3k77qcgywsin6pnt7u4l3va
 number_of_agents: 4
 deployment:
   agent:
@@ -112,6 +112,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 1:
   models:
     benchmark_tool:
@@ -176,6 +178,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 2:
   models:
     benchmark_tool:
@@ -240,6 +244,8 @@ extra:
         validate_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 3:
   models:
     benchmark_tool:
@@ -308,6 +314,8 @@ extra:
         history_check_timeout: 1205
         centaur_id_to_secrets: ${CENTAUR_ID_TO_SECRETS:str:{}}
         max_points_per_period: ${MAX_POINTS_PER_PERIOD:int:5000}
+        openai_call_window_size: ${OPENAI_CALL_WINDOW_SIZE:float:3600.0}
+        openai_calls_allowed_in_window: ${OPENAI_CALLS_ALLOWED_IN_WINDOW:int:100}
 ---
 public_id: valory/ledger:0.19.0
 type: connection

packages/valory/skills/impact_evaluator_abci/composition.py

@@ -51,8 +51,9 @@
     DecisionMakingAbci.FinishedDecisionMakingReadManualPointsRound: CeramicReadAbci.StreamReadRound,
     DecisionMakingAbci.FinishedDecisionMakingScoreRound: GenericScoringAbci.GenericScoringRound,
     DecisionMakingAbci.FinishedDecisionMakingDoneRound: ResetAndPauseAbci.ResetAndPauseRound,
-    GenericScoringAbci.FinishedGenericScoringRound: TwitterScoringAbci.TwitterCollectionRound,
+    GenericScoringAbci.FinishedGenericScoringRound: TwitterScoringAbci.OpenAICallCheckRound,
     TwitterScoringAbci.FinishedTwitterScoringRound: DynamicNFTAbci.TokenTrackRound,
+    TwitterScoringAbci.FinishedTwitterScoringWithAPIErrorRound: DynamicNFTAbci.TokenTrackRound,
     DynamicNFTAbci.FinishedTokenTrackRound: DecisionMakingAbci.DecisionMakingRound,
     LLMAbciApp.FinishedLLMRound: DecisionMakingAbci.DecisionMakingRound,
     TwitterWriteAbciApp.FinishedTwitterWriteRound: DecisionMakingAbci.DecisionMakingRound,

packages/valory/skills/impact_evaluator_abci/skill.yaml

@@ -8,7 +8,7 @@ aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
   __init__.py: bafybeigmhxenrqb2ysjhcm2au2lang4abyny6irkzqqg4dtclz6net6zpy
   behaviours.py: bafybeifhoufesr4r4re65mxjadqasjymvz6tkc25xn5yyn473yunhjiufe
-  composition.py: bafybeifvic3nzwujprattfcyxzxzp7g76x4ye24tte5p6rpvv6oe7sjzgm
+  composition.py: bafybeicjgndv2v43wclaoldmhpgugajsutqjfbhwbw67cmrli7z3eqpxne
   dialogues.py: bafybeigjknz4qqynbsltjje46gidg4rftsqw6ybjwegz24wetmycutpzh4
   fsm_specification.yaml: bafybeigfsjmxb7rbaup5ozfw2r54zh3qzlc3aeuozdsnegavbwf37bhpjy
   handlers.py: bafybeidkli6fphcmdgwsys4lkyf3fx6fbawet4nt2pnixfypzijhg6b3ze
@@ -26,7 +26,7 @@ skills:
 - valory/abstract_round_abci:0.1.0:bafybeigxjcci53vwytymzlhr37436yvenh7jup4astrn7dgyixo24aq2pq
 - valory/ceramic_read_abci:0.1.0:bafybeidruvxgpbggchuvlnssuswdxuoz2ep6sjvzzqjeyckybb6gpjx3ia
 - valory/generic_scoring_abci:0.1.0:bafybeiemmcnduhdj427kgafkxcbn2rfhaihabtpda34yqnks7utm3g4xiq
-- valory/twitter_scoring_abci:0.1.0:bafybeic2kz72uhkiaiwc6hactml74zl6hsgk3o4suikqvaeuth2664ctju
+- valory/twitter_scoring_abci:0.1.0:bafybeif63wgrs2ckqwfpfpfkrhrmhuykapqobmvry4a2dpfkpytnm4raam
 - valory/ceramic_write_abci:0.1.0:bafybeigtkv57h5suaqqpjfknimfegbidovxna7cok7znsgzrfb4nftqvay
 - valory/dynamic_nft_abci:0.1.0:bafybeie5vfscnjpvuwqe5vk7i2rzzlqciojswdg7uh34fjnuk3bspca5zq
 - valory/registration_abci:0.1.0:bafybeibc4kczqbh23sc6tufrzn3axmhp3vjav7fa3u6cnpvolrbbc2fd7i
@@ -169,6 +169,8 @@ models:
       twitter_max_pages: 10
       twitter_search_endpoint: 2/tweets/search/recent?
       twitter_search_args: query=%23olas&tweet.fields=author_id,created_at&user.fields=name&expansions=author_id&max_results=100&since_id={since_id}
+      openai_call_window_size: 3600.0
+      openai_calls_allowed_in_window: 100
       max_points_per_period: 5000
       tx_timeout: 10.0
       whitelist_api_key: null

packages/valory/skills/twitter_scoring_abci/behaviours.py

@@ -45,12 +45,14 @@
 from packages.valory.skills.twitter_scoring_abci.models import Params, SharedState
 from packages.valory.skills.twitter_scoring_abci.payloads import (
     DBUpdatePayload,
+    OpenAICallCheckPayload,
     TweetEvaluationPayload,
     TwitterCollectionPayload,
 )
 from packages.valory.skills.twitter_scoring_abci.prompts import tweet_evaluation_prompt
 from packages.valory.skills.twitter_scoring_abci.rounds import (
     DBUpdateRound,
+    OpenAICallCheckRound,
     SynchronizedData,
     TweetEvaluationRound,
     TwitterCollectionRound,
@@ -79,6 +81,34 @@ def params(self) -> Params:
         return cast(Params, super().params)
 
 
+class OpenAICallCheckBehaviour(TwitterScoringBaseBehaviour):
+    """TwitterCollectionBehaviour"""
+
+    matching_round: Type[AbstractRound] = OpenAICallCheckRound
+
+    def async_act(self) -> Generator:
+        """Do the act, supporting asynchronous execution."""
+        with self.context.benchmark_tool.measure(self.behaviour_id).local():
+            current_time = cast(
+                SharedState, self.context.state
+            ).round_sequence.last_round_transition_timestamp.timestamp()
+            # Reset the window if the window expired before checking
+            self.params.openai_calls.reset(current_time=current_time)
+            if self.params.openai_calls.max_tweets_reached():
+                content = OpenAICallCheckRound.API_CALL_EXCEEDED
+            else:
+                content = OpenAICallCheckRound.API_CALL_REMAINING
+        with self.context.benchmark_tool.measure(self.behaviour_id).consensus():
+            yield from self.send_a2a_transaction(
+                payload=OpenAICallCheckPayload(
+                    sender=self.context.agent_address,
+                    content=content,
+                )
+            )
+            yield from self.wait_until_round_end()
+        self.set_done()
+
+
 class TwitterCollectionBehaviour(TwitterScoringBaseBehaviour):
     """TwitterCollectionBehaviour"""
 
@@ -389,7 +419,7 @@ def evaluate_tweet(self, text: str) -> Generator[None, None, int]:
             request_llm_message, llm_dialogue
         )
         data = llm_response_message.value
-
+        self.params.openai_calls.increase_call_count()
         self.context.logger.info(f"Got tweet evaluation: {repr(data)}")
 
         points = DEFAULT_TWEET_POINTS

packages/valory/skills/twitter_scoring_abci/models.py

@@ -19,7 +19,8 @@
 
 """This module contains the shared state for the abci skill of TwitterScoringAbciApp."""
 
-from typing import Any
+from datetime import datetime
+from typing import Any, Optional
 
 from packages.valory.skills.abstract_round_abci.models import BaseParams
 from packages.valory.skills.abstract_round_abci.models import (
@@ -38,9 +39,45 @@ class SharedState(BaseSharedState):
     abci_app_cls = TwitterScoringAbciApp
 
 
+class OpenAICalls:
+    """OpenAI call window."""
+
+    def __init__(
+        self,
+        openai_call_window_size: float,
+        openai_calls_allowed_in_window: int,
+    ) -> None:
+        """Initialize object."""
+        self._calls_made_in_window = 0
+        self._calls_allowed_in_window = openai_calls_allowed_in_window
+        self._call_window_size = openai_call_window_size
+        self._call_window_start = datetime.now().timestamp()
+
+    def increase_call_count(self) -> None:
+        """Increase call count."""
+        self._calls_made_in_window += 1
+
+    def has_window_expired(self, current_time: float) -> bool:
+        """Increase tweet count."""
+        return current_time > (self._tweet_window_start + self._call_window_size)
+
+    def max_calls_reached(self) -> bool:
+        """Increase tweet count."""
+        return self._calls_made_in_window >= self._calls_allowed_in_window
+
+    def reset(self, current_time: float) -> None:
+        """Reset the window if required.."""
+        if not self.has_window_expired(current_time=current_time):
+            return
+        self._tweets_made_in_window = 0
+        self._tweet_window_start = current_time
+
+
 class Params(BaseParams):
     """Parameters."""
 
+    _current_tweet_window_start: Optional[float]
+
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         """Initialize the parameters object."""
         self.twitter_api_base = self._ensure("twitter_api_base", kwargs, str)
@@ -60,6 +97,15 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self.tweet_evaluation_round_timeout = self._ensure(
             "tweet_evaluation_round_timeout", kwargs, float
         )
+        self.openai_calls = OpenAICalls(
+            openai_call_window_size=self._ensure(
+                "openai_call_window_size", kwargs, float
+            ),
+            openai_calls_allowed_in_window=self._ensure(
+                "openai_calls_allowed_in_window", kwargs, int
+            ),
+        )
+
         super().__init__(*args, **kwargs)
 
 

packages/valory/skills/twitter_scoring_abci/payloads.py

@@ -43,3 +43,10 @@ class DBUpdatePayload(BaseTxPayload):
     """Represent a transaction payload for the DBUpdateRound."""
 
     content: str
+
+
+@dataclass(frozen=True)
+class OpenAICallCheckPayload(BaseTxPayload):
+    """Represent a transaction payload for the DBUpdateRound."""
+
+    content: str

packages/valory/skills/twitter_scoring_abci/rounds.py

@@ -89,6 +89,28 @@ def latest_mention_tweet_id(self) -> dict:
         return cast(dict, self.db.get_strict("latest_mention_tweet_id"))
 
 
+class OpenAICallCheckRound(CollectSameUntilThresholdRound):
+    """OpenAICallCheckRound"""
+
+    payload_class = TwitterCollectionPayload
+    synchronized_data_class = SynchronizedData
+
+    API_CALL_EXCEEDED = "API_CALLS_EXCEEDED"
+    API_CALL_REMAINING = "API_CALL_REMAINING"
+
+    def end_block(self) -> Optional[Tuple[BaseSynchronizedData, Event]]:
+        """Process the end of the block."""
+        if self.threshold_reached:
+            if self.most_voted_payload == self.API_CALL_EXCEEDED:
+                return self.synchronized_data, Event.API_ERROR
+            return self.synchronized_data, Event.DONE
+        if not self.is_majority_possible(
+            self.collection, self.synchronized_data.nb_participants
+        ):
+            return self.synchronized_data, Event.NO_MAJORITY
+        return None
+
+
 class TwitterCollectionRound(CollectSameUntilThresholdRound):
     """TwitterCollectionRound"""
 
@@ -213,12 +235,20 @@ class FinishedTwitterScoringRound(DegenerateRound):
     """FinishedTwitterScoringRound"""
 
 
+class FinishedTwitterScoringWithAPIErrorRound(DegenerateRound):
+    """FinishedTwitterScoringRound"""
+
+
 class TwitterScoringAbciApp(AbciApp[Event]):
     """TwitterScoringAbciApp"""
 
-    initial_round_cls: AppState = TwitterCollectionRound
-    initial_states: Set[AppState] = {TwitterCollectionRound}
+    initial_round_cls: AppState = OpenAICallCheckRound
+    initial_states: Set[AppState] = {OpenAICallCheckRound}
     transition_function: AbciAppTransitionFunction = {
+        OpenAICallCheckRound: {
+            Event.DONE: TwitterCollectionRound,
+            Event.API_ERROR: FinishedTwitterScoringWithAPIErrorRound,
+        },
         TwitterCollectionRound: {
             Event.DONE: TweetEvaluationRound,
             Event.API_ERROR: TwitterCollectionRound,

packages/valory/skills/twitter_scoring_abci/skill.yaml

@@ -8,15 +8,15 @@ license: Apache-2.0
 aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
   __init__.py: bafybeifudgakkjoyahuewp2o4gvqayw7nsgpyxw2ayrpgmzexurh2xomaq
-  behaviours.py: bafybeidsbxru6el5al67s27gfibrkusrqodb6unpb5vkjjwukrrvk3t6yu
+  behaviours.py: bafybeigwsbwlvdlviyzbkw4wh4p3f27soh7q6ntocyjbq6gypun5pddr3m
   ceramic_db.py: bafybeicusdonrdq6kirgkpdqmi3a6kmeal4nctnm5ozjqf5s5se6jpitjm
   dialogues.py: bafybeibdqzn37hbo2cq4skww4uh2zvvsjyaxxvdhxisefbdvmjp7rh53si
   fsm_specification.yaml: bafybeiedr7ojvlyriotk6za7fnt4ptnhxklehznn3selyprbehj7tb4kem
   handlers.py: bafybeid3nqvcyotqj5g5hlgrz57nf7vpjysmgvsxe3p7644f4z5dcwqn6u
-  models.py: bafybeifvmpurh6w4ml27u4kvcjdsh62ldmx3oy5hrfzbqa4ifikvnxzh7m
-  payloads.py: bafybeiapqgm3uscsl2gghy6fe22jceta4d5suujdlux233c37fyqydjp5i
+  models.py: bafybeigpjxkhgsu7c542bp2n6opbl5oybtg2vxu4xhkonalsogq4edmzbe
+  payloads.py: bafybeigqmoljkexbkldybrk63aqpcnfxftys4lfttl4zyzrpds7seqerw4
   prompts.py: bafybeieiuqn427bgwfnzynxf3vtqfpvmqqscs5tyw4oibfofwropifotke
-  rounds.py: bafybeidfkps2sqegmbqqskimxkzmq3c3sme3zgsbdxkoyjlccvyqaooi4u
+  rounds.py: bafybeihcooulvmh3yogx5b5valbuztqh76hgdhbb2ctumkgd4rak7hvlnq
   tests/__init__.py: bafybeidwzzd4ejsyf3aryd5kmrvd63h7ajgqyrxphmfaacvpjnneacejay
   tests/test_behaviours.py: bafybeidhcwz3gwjbwlktbgkafps3wjiyuipewtxa3hponqafgkecwb76we
   tests/test_ceramic_db.py: bafybeif2v7btjphbqabq6qdmfeyweg765seon74acg5vrrvznivzg2prey
@@ -139,6 +139,8 @@ models:
       twitter_max_pages: 10
       twitter_search_endpoint: 2/tweets/search/recent?
       twitter_search_args: query=%23olas&tweet.fields=author_id,created_at&user.fields=name&expansions=author_id&max_results=100&since_id={since_id}
+      openai_call_window_size: 3600.0
+      openai_calls_allowed_in_window: 100
       max_points_per_period: 5000
       validate_timeout: 1205
       use_termination: false