fix(formatter): decode tweet text HTML entities

package.json

@@ -32,6 +32,7 @@
     "@types/node": "^20.9.0",
     "cli-spinners": "^2.9.1",
     "dotenv": "^16.3.1",
+    "html-entities": "^2.4.0",
     "html-escaper": "^3.0.3",
     "masto": "^6.4.2",
     "mime": "^3.0.0",

src/helpers/tweet/__tests__/format-tweet-text.spec.ts

@@ -3,6 +3,17 @@ import { Tweet } from "@the-convocation/twitter-scraper";
 import { formatTweetText } from "../format-tweet-text.js";
 
 describe("formatTweetText", () => {
+  it("should decode html entities from the text", () => {
+    const tweet: Tweet = {
+      text: "one elephant & one mammoth make two animals, right?< > & " ' © ® ™ ¢ £ € ¥ § ° ± × ÷",
+      urls: [],
+    } as unknown as Tweet;
+    const result = formatTweetText(tweet);
+    expect(result).toStrictEqual(
+      "one elephant & one mammoth make two animals, right?< > & \" ' © ® ™ ¢ £ € ¥ § ° ± × ÷",
+    );
+  });
+
   describe("when the tweet has links", () => {
     it("should remove the t.co links from the text", () => {
       const tweet: Tweet = {

src/helpers/tweet/format-tweet-text.ts

@@ -1,4 +1,5 @@
 import { Tweet } from "@the-convocation/twitter-scraper";
+import { decode } from "html-entities";
 
 export const formatTweetText = (tweet: Tweet): string => {
   let text = tweet.text ?? "";
@@ -11,6 +12,9 @@ export const formatTweetText = (tweet: Tweet): string => {
   // Remove medias t.co links
   text = text.replaceAll(/https:\/\/t\.co\/\w+/g, "");
 
+  // Replace HTML entities with their unicode equivalent
+  text = decode(text);
+
   // Return formatted
   return text.trim();
 };