feat(store-indexer): add api version

- Add an `/api/2/logs` route and deprecate the previous version (which remains in operation under the current route `/api/logs` for backwards compatibility)
- Intentionally repeat the code verbatim (but without the unwanted filters) so that future modifications do not break third party uses of the v1 route.

packages/store-indexer/src/bin/postgres-frontend.ts

@@ -9,7 +9,8 @@ import { drizzle } from "drizzle-orm/postgres-js";
 import postgres from "postgres";
 import { frontendEnvSchema, parseEnv } from "./parseEnv";
 import { createQueryAdapter } from "../postgres/deprecated/createQueryAdapter";
-import { apiRoutes } from "../postgres/apiRoutes";
+import { apiRoutesV1 } from "../postgres/apiRoutes/v1";
+import { apiRoutesV2 } from "../postgres/apiRoutes/v2";
 import { sentry } from "../koa-middleware/sentry";
 import { healthcheck } from "../koa-middleware/healthcheck";
 import { helloWorld } from "../koa-middleware/helloWorld";
@@ -42,7 +43,8 @@ server.use(
   }),
 );
 server.use(helloWorld());
-server.use(apiRoutes(database));
+server.use(apiRoutesV1(database));
+server.use(apiRoutesV2(database));
 
 server.use(
   createKoaMiddleware({

packages/store-indexer/src/bin/sqlite-indexer.ts

@@ -17,7 +17,8 @@ import { combineLatest, filter, first } from "rxjs";
 import { frontendEnvSchema, indexerEnvSchema, parseEnv } from "./parseEnv";
 import { healthcheck } from "../koa-middleware/healthcheck";
 import { helloWorld } from "../koa-middleware/helloWorld";
-import { apiRoutes } from "../sqlite/apiRoutes";
+import { apiRoutesV1 } from "../sqlite/apiRoutes/v1";
+import { apiRoutesV2 } from "../sqlite/apiRoutes/v2";
 import { sentry } from "../koa-middleware/sentry";
 import { metrics } from "../koa-middleware/metrics";
 
@@ -145,7 +146,8 @@ server.use(
   }),
 );
 server.use(helloWorld());
-server.use(apiRoutes(database));
+server.use(apiRoutesV1(database));
+server.use(apiRoutesV2(database));
 
 server.use(
   createKoaMiddleware({

packages/store-indexer/src/postgres/apiRoutes.ts → packages/store-indexer/src/postgres/apiRoutes/v1.ts

@@ -4,15 +4,21 @@ import Router from "@koa/router";
 import compose from "koa-compose";
 import { input } from "@latticexyz/store-sync/indexer-client";
 import { schemasTable } from "@latticexyz/store-sync";
-import { queryLogs } from "./queryLogs";
-import { recordToLog } from "./recordToLog";
-import { debug, error } from "../debug";
+import { queryLogs } from "../queryLogs";
+import { recordToLog } from "../recordToLog";
+import { debug, error } from "../../debug";
 import { createBenchmark } from "@latticexyz/common";
-import { compress } from "../koa-middleware/compress";
+import { compress } from "../../koa-middleware/compress";
 
-export function apiRoutes(database: Sql): Middleware {
+/**
+ * @deprecated
+ * Please keep this implementation unchanged in order to maintain
+ * backwards compatibility for third parties.
+ */
+export function apiRoutesV1(database: Sql): Middleware {
   const router = new Router();
 
+  // Unversioned path for backwards compatibility
   router.get("/api/logs", compress(), async (ctx) => {
     const benchmark = createBenchmark("postgres:logs");
     let options: ReturnType<typeof input.parse>;

packages/store-indexer/src/postgres/apiRoutes/v2.ts

@@ -0,0 +1,79 @@
+import { Sql } from "postgres";
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { queryLogs } from "../queryLogs";
+import { recordToLog } from "../recordToLog";
+import { debug, error } from "../../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../../koa-middleware/compress";
+
+export function apiRoutesV2(database: Sql): Middleware {
+  const router = new Router();
+
+  router.get("/api/2/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("postgres:logs:v2");
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (e) {
+      ctx.status = 400;
+      ctx.set("Content-Type", "application/json");
+      ctx.body = JSON.stringify(e);
+      debug(e);
+      return;
+    }
+
+    try {
+      const records = await queryLogs(database, options ?? {}).execute();
+      benchmark("query records");
+      const logs = records.map(recordToLog);
+      benchmark("map records to logs");
+
+      // Ideally we would immediately return an error if the request is for a Store that the indexer
+      // is not configured to index. Since we don't have easy access to this information here,
+      // we return an error if there are no logs found for a given Store, since that would never
+      // be the case for a Store that is being indexed (since there would at least be records for the
+      // Tables table with tables created during Store initialization).
+      if (records.length === 0) {
+        ctx.status = 404;
+        ctx.body = "no logs found";
+        error(
+          `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify(
+            options.filters,
+          )}`,
+        );
+        return;
+      }
+
+      const blockNumber = records[0].chainBlockNumber;
+      ctx.status = 200;
+
+      // max age is set to several multiples of the uncached response time (currently ~10s, but using 60s for wiggle room) to ensure only ~one origin request at a time
+      // and stale-while-revalidate below means that the cache is refreshed under the hood while still responding fast (cached)
+      const maxAgeSeconds = 60 * 5;
+      // we set stale-while-revalidate to the time elapsed by the number of blocks we can fetch from the RPC in the same amount of time as an uncached response
+      // meaning it would take ~the same about of time to get an uncached response from the origin as it would to catch up from the currently cached response
+      // if an uncached response takes ~10 seconds, we have ~10s to catch up, so let's say we can do enough RPC calls to fetch 4000 blocks
+      // with a block per 2 seconds, that means we can serve a stale/cached response for 8000 seconds before we should require the response be returned by the origin
+      const staleWhileRevalidateSeconds = 4000 * 2;
+
+      ctx.set(
+        "Cache-Control",
+        `public, max-age=${maxAgeSeconds}, stale-while-revalidate=${staleWhileRevalidateSeconds}`,
+      );
+
+      ctx.set("Content-Type", "application/json");
+      ctx.body = JSON.stringify({ blockNumber, logs });
+    } catch (e) {
+      ctx.status = 500;
+      ctx.set("Content-Type", "application/json");
+      ctx.body = JSON.stringify(e);
+      error(e);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}

packages/store-indexer/src/sqlite/apiRoutes.ts → packages/store-indexer/src/sqlite/apiRoutes/v1.ts

@@ -3,16 +3,22 @@ import Router from "@koa/router";
 import compose from "koa-compose";
 import { input } from "@latticexyz/store-sync/indexer-client";
 import { schemasTable, tablesWithRecordsToLogs } from "@latticexyz/store-sync";
-import { debug } from "../debug";
+import { debug } from "../../debug";
 import { createBenchmark } from "@latticexyz/common";
-import { compress } from "../koa-middleware/compress";
-import { getTablesWithRecords } from "./getTablesWithRecords";
+import { compress } from "../../koa-middleware/compress";
+import { getTablesWithRecords } from "../getTablesWithRecords";
 import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
 
+/**
+ * @deprecated
+ * Please keep this implementation unchanged in order to maintain
+ * backwards compatibility for third parties.
+ */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware {
+export function apiRoutesV1(database: BaseSQLiteDatabase<"sync", any>): Middleware {
   const router = new Router();
 
+  // Unversioned path for backwards compatibility
   router.get("/api/logs", compress(), async (ctx) => {
     const benchmark = createBenchmark("sqlite:logs");
 

packages/store-indexer/src/sqlite/apiRoutes/v2.ts

@@ -0,0 +1,47 @@
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { tablesWithRecordsToLogs } from "@latticexyz/store-sync";
+import { debug } from "../../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../../koa-middleware/compress";
+import { getTablesWithRecords } from "../getTablesWithRecords";
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function apiRoutesV2(database: BaseSQLiteDatabase<"sync", any>): Middleware {
+  const router = new Router();
+
+  router.get("/api/2/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("sqlite:logs:v2");
+
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (error) {
+      ctx.status = 400;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+      return;
+    }
+
+    try {
+      benchmark("parse config");
+      const { blockNumber, tables } = getTablesWithRecords(database, options);
+      benchmark("query tables with records");
+      const logs = tablesWithRecordsToLogs(tables);
+      benchmark("convert records to logs");
+
+      ctx.body = JSON.stringify({ blockNumber: blockNumber?.toString() ?? "-1", logs });
+      ctx.status = 200;
+    } catch (error) {
+      ctx.status = 500;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}