Skip to content

Commit

Permalink
fix(cache): assume the sync is useless if latest post is already cach…
Browse files Browse the repository at this point in the history
…ed (#99)
  • Loading branch information
louisgrasset authored Oct 27, 2023
1 parent df5968f commit 24f5003
Showing 1 changed file with 17 additions and 4 deletions.
21 changes: 17 additions & 4 deletions src/services/tweets-getter.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,28 @@ export const tweetsGetterService = async (
const tweets: Tweet[] = [];
const tweetsIds = twitterClient.getTweets(TWITTER_HANDLE, 200);

let hasRateLimit = false;
let hasRateLimitReached = false;
let latestTweetAlreadySynced = false;
let tweetsCount = 0;
for await (const tweet of tweetsIds) {
const rateLimitTimeout = setTimeout(
() => (hasRateLimit = true),
() => (hasRateLimitReached = true),
1000 * API_RATE_LIMIT,
);
if (hasRateLimit || isTweetCached(tweet, cache)) {

if (
latestTweetAlreadySynced ||
hasRateLimitReached ||
isTweetCached(tweet, cache)
) {
continue;
}

// Skip posts sync if the latest one has already synced
if (tweetsCount === 0 && isTweetCached(tweet, cache)) {
latestTweetAlreadySynced = true;
}

const t: Tweet = {
...tweet,
timestamp: (tweet.timestamp ?? 0) * 1000,
Expand All @@ -59,9 +71,10 @@ export const tweetsGetterService = async (
tweets.unshift(eligibleTweet);
}
clearTimeout(rateLimitTimeout);
tweetsCount++;
}

if (hasRateLimit) {
if (hasRateLimitReached) {
log.warn(
`rate limit reached, more than ${API_RATE_LIMIT}s to fetch a single tweet`,
);
Expand Down

0 comments on commit 24f5003

Please sign in to comment.