feat: prevent unecessary sync (#102)

src/helpers/cache/migrations/helpers/prevent-migration-on-wrong-version.ts

@@ -0,0 +1,6 @@
+export const preventMigrationOnWrongVersion = (
+  outdatedCache: NonNullable<unknown>,
+  requiredVersion: string,
+) =>
+  (outdatedCache as NonNullable<unknown & { version: string }>).version !==
+  requiredVersion;

src/helpers/tweet/__tests__/tweet-formatter.spec.ts

@@ -0,0 +1,25 @@
+import { Tweet } from "@the-convocation/twitter-scraper";
+
+import { tweetFormatter } from "../tweet-formatter.js";
+
+jest.mock("../format-tweet-text.js", () => {
+  return {
+    formatTweetText: jest
+      .fn()
+      .mockImplementation((t: Tweet) => `formatted:${t.text}`),
+  };
+});
+
+describe("tweetFormatter", () => {
+  it("should properly format the give tweet", () => {
+    const result = tweetFormatter({
+      text: "text",
+      timestamp: 966236400,
+    } as Tweet);
+
+    expect(result).toStrictEqual({
+      text: "formatted:text",
+      timestamp: 966236400000,
+    });
+  });
+});

src/helpers/tweet/index.ts

@@ -3,3 +3,4 @@ export * from "./is-tweet-cached.js";
 export * from "./keep-recent-tweets.js";
 export * from "./keep-self-quotes.js";
 export * from "./keep-self-replies.js";
+export * from "./tweet-formatter.js";

src/helpers/tweet/tweet-formatter.ts

@@ -0,0 +1,11 @@
+import { Tweet } from "@the-convocation/twitter-scraper";
+
+import { formatTweetText } from "./format-tweet-text.js";
+
+export const tweetFormatter = (tweet: Tweet): Tweet => {
+  return {
+    ...tweet,
+    timestamp: (tweet.timestamp ?? 0) * 1000,
+    text: formatTweetText(tweet),
+  };
+};

src/services/tweets-getter.service.ts

@@ -5,7 +5,7 @@ import { API_RATE_LIMIT, TWITTER_HANDLE } from "../constants.js";
 import { getCache } from "../helpers/cache/index.js";
 import { oraPrefixer } from "../helpers/logs/ora-prefixer.js";
 import { getEligibleTweet } from "../helpers/tweet/get-eligible-tweet.js";
-import { formatTweetText, isTweetCached } from "../helpers/tweet/index.js";
+import { isTweetCached, tweetFormatter } from "../helpers/tweet/index.js";
 
 const pullContentStats = (tweets: Tweet[], title: string) => {
   const stats = {
@@ -34,53 +34,74 @@ export const tweetsGetterService = async (
   }).start();
   log.text = "filtering";
 
+  let preventPostsSynchronization = false;
+  const LATEST_TWEETS_COUNT = 5;
+
+  /**
+   * Synchronization optimization: prevent excessive API calls & potential rate-limiting
+   *
+   * Pull the ${LATEST_TWEETS_COUNT}, filter eligible ones.
+   * This optimization prevents the post sync if the latest eligible tweet is cached.
+   */
+  const latestTweets = twitterClient.getTweets(
+    TWITTER_HANDLE,
+    LATEST_TWEETS_COUNT,
+  );
+
+  for await (const latestTweet of latestTweets) {
+    if (!preventPostsSynchronization) {
+      // Only consider eligible tweets.
+      const tweet = await getEligibleTweet(tweetFormatter(latestTweet));
+
+      if (tweet) {
+        // If the latest eligible tweet is cached, mark sync as unneeded.
+        if (isTweetCached(tweet, cache)) {
+          preventPostsSynchronization = true;
+        }
+        // If the latest tweet is not cached,
+        // skip the current optimization and go to synchronization step.
+        break;
+      }
+    }
+  }
+
   // Get tweets from API
   const tweets: Tweet[] = [];
-  const tweetsIds = twitterClient.getTweets(TWITTER_HANDLE, 200);
-
-  let hasRateLimitReached = false;
-  let latestTweetAlreadySynced = false;
-  let tweetsCount = 0;
-  for await (const tweet of tweetsIds) {
-    const rateLimitTimeout = setTimeout(
-      () => (hasRateLimitReached = true),
-      1000 * API_RATE_LIMIT,
-    );
-
-    if (
-      latestTweetAlreadySynced ||
-      hasRateLimitReached ||
-      isTweetCached(tweet, cache)
-    ) {
-      continue;
-    }
 
-    // Skip posts sync if the latest one has already synced
-    if (tweetsCount === 0 && isTweetCached(tweet, cache)) {
-      latestTweetAlreadySynced = true;
-    }
+  if (preventPostsSynchronization) {
+    log.succeed("task finished (unneeded sync)");
+  } else {
+    const tweetsIds = twitterClient.getTweets(TWITTER_HANDLE, 200);
+
+    let hasRateLimitReached = false;
+    for await (const tweet of tweetsIds) {
+      const rateLimitTimeout = setTimeout(
+        () => (hasRateLimitReached = true),
+        1000 * API_RATE_LIMIT,
+      );
+
+      if (hasRateLimitReached || isTweetCached(tweet, cache)) {
+        continue;
+      }
 
-    const t: Tweet = {
-      ...tweet,
-      timestamp: (tweet.timestamp ?? 0) * 1000,
-      text: formatTweetText(tweet),
-    };
+      const t: Tweet = tweetFormatter(tweet);
 
-    const eligibleTweet = await getEligibleTweet(t);
-    if (eligibleTweet) {
-      tweets.unshift(eligibleTweet);
+      const eligibleTweet = await getEligibleTweet(t);
+      if (eligibleTweet) {
+        tweets.unshift(eligibleTweet);
+      }
+      clearTimeout(rateLimitTimeout);
+    }
+
+    if (hasRateLimitReached) {
+      log.warn(
+        `rate limit reached, more than ${API_RATE_LIMIT}s to fetch a single tweet`,
+      );
     }
-    clearTimeout(rateLimitTimeout);
-    tweetsCount++;
-  }
 
-  if (hasRateLimitReached) {
-    log.warn(
-      `rate limit reached, more than ${API_RATE_LIMIT}s to fetch a single tweet`,
-    );
+    log.succeed(pullContentStats(tweets, "tweets"));
+    log.succeed("task finished");
   }
-  log.succeed(pullContentStats(tweets, "tweets"));
-  log.succeed("task finished");
 
   return tweets;
 };