feat: add rate limit detection

louisgrasset · Oct 18, 2023 · 437535d · 437535d
1 parent b0ec30f
commit 437535d
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 1 deletion.
diff --git a/src/constants.ts b/src/constants.ts
@@ -37,3 +37,4 @@ export const DEBUG = (process.env.TOUITOMAMOUT_DEBUG || 'false') === 'true';
 export const DAEMON = (process.env.DAEMON || 'false') === 'true';
 export const DAEMON_PERIOD_MIN = parseInt(process.env.DAEMON_PERIOD_MIN ?? '7'); // Default 7 min
 export const VOID = '∅';
+export const API_RATE_LIMIT = parseInt(process.env.API_RATE_LIMIT ?? '60');
diff --git a/src/services/tweets-getter.service.ts b/src/services/tweets-getter.service.ts
@@ -1,7 +1,7 @@
 import { Scraper, Tweet } from '@the-convocation/twitter-scraper';
 import ora from 'ora';
 
-import { TWITTER_HANDLE } from '../constants.js';
+import { API_RATE_LIMIT, TWITTER_HANDLE } from '../constants.js';
 import { getCache } from '../helpers/cache/index.js';
 import { oraPrefixer } from '../helpers/logs/ora-prefixer.js';
 import { getEligibleTweet } from '../helpers/tweet/get-eligible-tweet.js';
@@ -27,7 +27,12 @@ export const tweetsGetterService = async (twitterClient: Scraper): Promise<Tweet
     const tweets: Tweet[] = [];
     const tweetsIds = twitterClient.getTweets(TWITTER_HANDLE, 200);
 
+    let hasRateLimit = true;
     for await(const tweet of tweetsIds) {
+        const rateLimitTimeout = setTimeout(() => hasRateLimit = true,1000 * API_RATE_LIMIT);
+        if(hasRateLimit) {
+            continue;
+        }
         if (tweet && !isTweetCached(tweet, cache)) {
             const t: Tweet = {
                 ...tweet,
@@ -57,8 +62,12 @@ export const tweetsGetterService = async (twitterClient: Scraper): Promise<Tweet
                 tweets.unshift(eligibleTweet);
             }
         }
+        clearTimeout(rateLimitTimeout);
     }
 
+    if(hasRateLimit) {
+        log.warn(`rate limit reached, more than ${API_RATE_LIMIT}s to fetch a single tweet`);
+    }
     log.succeed(pullContentStats(tweets, 'tweets'));
     log.succeed('task finished');