From 437535d6dcb3ae5096bb22b62a1355d46f30599e Mon Sep 17 00:00:00 2001
From: Louis Grasset <louisgrasset@protonmail.com>
Date: Wed, 18 Oct 2023 08:37:07 +0200
Subject: [PATCH] feat: add rate limit detection

---
 src/constants.ts                      |  1 +
 src/services/tweets-getter.service.ts | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/constants.ts b/src/constants.ts
index fb400c3..cf02c5e 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -37,3 +37,4 @@ export const DEBUG = (process.env.TOUITOMAMOUT_DEBUG || 'false') === 'true';
 export const DAEMON = (process.env.DAEMON || 'false') === 'true';
 export const DAEMON_PERIOD_MIN = parseInt(process.env.DAEMON_PERIOD_MIN ?? '7'); // Default 7 min
 export const VOID = '∅';
+export const API_RATE_LIMIT = parseInt(process.env.API_RATE_LIMIT ?? '60');
diff --git a/src/services/tweets-getter.service.ts b/src/services/tweets-getter.service.ts
index 1b356df..ccaefab 100644
--- a/src/services/tweets-getter.service.ts
+++ b/src/services/tweets-getter.service.ts
@@ -1,7 +1,7 @@
 import { Scraper, Tweet } from '@the-convocation/twitter-scraper';
 import ora from 'ora';
 
-import { TWITTER_HANDLE } from '../constants.js';
+import { API_RATE_LIMIT, TWITTER_HANDLE } from '../constants.js';
 import { getCache } from '../helpers/cache/index.js';
 import { oraPrefixer } from '../helpers/logs/ora-prefixer.js';
 import { getEligibleTweet } from '../helpers/tweet/get-eligible-tweet.js';
@@ -27,7 +27,12 @@ export const tweetsGetterService = async (twitterClient: Scraper): Promise<Tweet
     const tweets: Tweet[] = [];
     const tweetsIds = twitterClient.getTweets(TWITTER_HANDLE, 200);
 
+    let hasRateLimit = true;
     for await(const tweet of tweetsIds) {
+        const rateLimitTimeout = setTimeout(() => hasRateLimit = true,1000 * API_RATE_LIMIT);
+        if(hasRateLimit) {
+            continue;
+        }
         if (tweet && !isTweetCached(tweet, cache)) {
             const t: Tweet = {
                 ...tweet,
@@ -57,8 +62,12 @@ export const tweetsGetterService = async (twitterClient: Scraper): Promise<Tweet
                 tweets.unshift(eligibleTweet);
             }
         }
+        clearTimeout(rateLimitTimeout);
     }
 
+    if(hasRateLimit) {
+        log.warn(`rate limit reached, more than ${API_RATE_LIMIT}s to fetch a single tweet`);
+    }
     log.succeed(pullContentStats(tweets, 'tweets'));
     log.succeed('task finished');