feat(ipns-with-gw3): add ./bin/refresh-ipns-gw3.ts & lambda handler

thematters · Jan 10, 2024 · e9c911f · e9c911f
1 parent 34aecda
commit e9c911f
Show file tree

Hide file tree

Showing 8 changed files with 1,266 additions and 374 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -18,5 +18,9 @@
     "sourceType": "module"
   },
   "plugins": ["@typescript-eslint"],
-  "rules": {}
+  "rules": {
+    "no-unused-vars": "off",
+    "@typescript-eslint/no-unused-vars": "off",
+    "@typescript-eslint/no-explicit-any": "off"
+  }
 }
diff --git a/bin/refresh-ipns-gw3.ts b/bin/refresh-ipns-gw3.ts
@@ -10,22 +10,25 @@ import {
 import slugify from "@matters/slugify";
 
 import {
-  gw3Client,
+  // gw3Client,
   refreshPinLatest,
   refreshIPNSFeed,
+  purgeIPNS,
 } from "../lib/refresh-ipns-gw3.js";
 import { AuthorFeed } from "../lib/author-feed-ipns.js";
 import { ipfsPool } from "../lib/ipfs-servers.js";
 import { dbApi, Item } from "../lib/db.js";
 
 async function main() {
   const args = process.argv.slice(2);
-  let mode: "publishIPNS" | "publishIPFS" | "uploadPinning" = "publishIPNS";
+  let mode: "publishIPNS" | "publishIPFS" | "uploadPinning" | "purgeIPNS" =
+    "publishIPNS";
 
   switch (args?.[0]) {
     case "--publishIPNS":
     case "--publishIPFS":
     case "--uploadPinning":
+    case "--purgeIPNS": // remove IPNS from gw3 for too old no updating ones
       mode = args?.[0].substring(2) as any;
       args.shift();
       break;
@@ -61,12 +64,14 @@ async function main() {
     const [ipnsKeyRec] = await dbApi.getUserIPNSKey(author.id);
     console.log(new Date(), "get user ipns:", ipnsKeyRec);
 
-    const feed = new AuthorFeed(
+    // const feed = new AuthorFeed({ author, ipnsKey: ipnsKeyRec?.ipnsKey, webfHost, drafts, articles });
+    const feed = new AuthorFeed({
       author,
-      ipnsKeyRec?.ipnsKey,
-      drafts.slice(0, 1),
-      articles.slice(0, 1)
-    );
+      ipnsKey: ipnsKeyRec?.ipnsKey,
+      // webfHost, // this is a fake one; no need for single article publishing
+      drafts, // .slice(0, 1),
+      articles, // .slice(0, 1)
+    });
 
     // console.log(new Date(), "get author feed:", feed);
     await feed.loadData();
@@ -96,27 +101,33 @@ async function main() {
     const offset = parseInt(args?.[1] ?? "0");
     await refreshPinLatest({ limit, offset });
     return;
+  } else if (mode === "purgeIPNS") {
+    await purgeIPNS({ skip: 10000 });
+    return;
   }
 
   let forceReplace = false;
   let useMattersIPNS: boolean | undefined;
+  let webfHost: string | undefined;
 
   // publish IPNS mode
   console.log(new Date(), "running with:", args);
-  switch (args?.[0]) {
-    case "--forceReplace":
-      forceReplace = true;
-      args.shift();
-      break;
-    case "--useMattersIPNS":
-    case "--useMattersIPNS=true":
-      useMattersIPNS = true;
-      args.shift();
-      break;
-    case "--useMattersIPNS=false":
-      useMattersIPNS = false;
-      args.shift();
-      break;
+  while (args?.[0]?.startsWith("--")) {
+    switch (args?.[0]?.split("=")?.[0]) {
+      case "--forceReplace":
+        forceReplace = true;
+        break;
+      case "--useMattersIPNS":
+        // case "--useMattersIPNS=true": useMattersIPNS = true;
+        useMattersIPNS = !(args?.[0]?.split("=")?.[1] === "false");
+        break;
+      // case "--useMattersIPNS=false": useMattersIPNS = false; break;
+      case "--webfHost":
+        webfHost = args?.[0]?.split("=")?.[1] as string;
+        console.log(new Date(), "set webfHost:", webfHost);
+        break;
+    }
+    args.shift();
   }
 
   // await testPinning();
@@ -128,6 +139,7 @@ async function main() {
     limit,
     forceReplace,
     useMattersIPNS,
+    webfHost,
   });
   console.log(new Date(), `refreshIPNSFeed res:`, res);
   if (res && res.missing <= res.limit / 10) return;
@@ -140,6 +152,7 @@ async function main() {
       limit: 10,
       forceReplace,
       useMattersIPNS,
+      webfHost,
     });
     console.log(new Date(), `try again refreshIPNSFeed res:`, res);
   }

diff --git a/handlers/refresh-ipns-gw3.ts b/handlers/refresh-ipns-gw3.ts
@@ -1,6 +1,6 @@
 import { Context, APIGatewayProxyResult, APIGatewayEvent } from "aws-lambda";
 
-import { refreshIPNSFeed } from "../lib/refresh-ipns-gw3.js";
+import { purgeIPNS, refreshIPNSFeed } from "../lib/refresh-ipns-gw3.js";
 import { dbApi, Item } from "../lib/db.js";
 
 export const handler = async (
@@ -40,7 +40,8 @@ export const handler = async (
     names = authors.map(({ userName }) => userName).filter(Boolean);
     console.log(
       new Date(),
-      `got latest author '${names}' fromrecent authors:`,
+      `got latest recent ${authors.length} authors:`,
+      names,
       authors
     );
   }
@@ -70,6 +71,8 @@ export const handler = async (
     }
   }
 
+  await purgeIPNS();
+
   return {
     statusCode: 200,
     body: JSON.stringify({