feat: upgrade twitter api

src/bff.ts

@@ -79,7 +79,7 @@ createExpressApp().then(async ({ app, server, cache }) => {
   const getTwitterAggregateCache = cacher.interval(cache, {
     key: TunnelModule.TwitterAggregate,
     ttl: hours(12),
-    interval: hours(2.5),
+    interval: hours(1.6),
     retry: minutes(10),
     getter: getTwitterAggregate
   })

src/server/getters/twitter/index.ts

@@ -45,6 +45,12 @@ export interface TwitterAggregate {
 }
 
 export const getTwitterAggregate = async (): Promise<TwitterAggregate> => {
+  return getSotweTwitterAggregate(IDENTITIES.TWITTER_USER_NAME)
+}
+
+// MARK: Nitter is over https://nitter.cz/
+// https://github.com/zedeus/nitter/issues/1171
+export const getTwitterAggregateLegacy = async (): Promise<TwitterAggregate> => {
   const [sotwe, nitter] = await Promise.allSettled([
     getSotweTwitterAggregate(IDENTITIES.TWITTER_USER_NAME),
     getNitterTweets(IDENTITIES.TWITTER_USER_NAME)

src/server/getters/twitter/sotwe-api.ts

@@ -88,24 +88,28 @@ export interface SotweAggregate {
 
 // Don't try to simulate a browser request, it will be blocked by Cloudflare.
 // Can't make requests based on headless browsers because it takes up too much memory
-// So one has to look for third party online services, here are some of the available crawler services:
+// So one has to look for third party online services, here are some of the available scraper services:
 // - https://dashboard.scrape.do
+// - https://dashboard.scraperapi.com
 // - https://apilayer.com/marketplace/adv_scraper-api
 // - ...
 const fetchSotweAggregate = async (twitterUsername: string): Promise<SotweAggregate> => {
   try {
     const target = `https://api.sotwe.com/v3/user/${twitterUsername}`
-    const scraper = `http://api.scrape.do/?token=${SOTWE_SCRAPER_TOKEN}&url=${target}`
+    // const _scrape_do = `http://api.scrape.do/?token=${SOTWE_SCRAPER_TOKEN}&url=${target}`
+    const _scraper_api = `https://api.scraperapi.com/?api_key=${SOTWE_SCRAPER_TOKEN}&url=${target}`
     // To avoid wasting request credits, tokens are not used in development environments
-    const response = await axios.get<SotweAggregate>(isNodeDev ? target : scraper, { timeout: 18000 })
+    const response = await axios.get<SotweAggregate>(isNodeDev ? target : _scraper_api, { timeout: 28000 })
     return response.data
   } catch (error: unknown) {
     throw isAxiosError(error) ? error.toJSON() : error
   }
 }
 
 const improveSotweTweet = (tweet: SotweTweet): string => {
-  let result = tweet.text.replaceAll('\n', ' ').replace(/ +/g, ' ')
+  // remove new lines and multiple spaces
+  // replace unknown characters
+  let result = tweet.text.replaceAll('\n', ' ').replace(/ +/g, ' ').replaceAll('�', '_')
   // remove media urls
   tweet.mediaEntities?.forEach((media) => {
     result = result.replace(media.url, '')